mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
test(crash-reporting): cover the cgroup/PSI sysfs layer and unpin it from the CI host
The reader test doubles skipped every line that actually opens a file: with
resolveCgroupV2MemoryDir, readLinuxCgroupMemoryLimitFromSysfs and
readLinuxMemoryPressureStallFromProcfs each throwing on entry, all 306
crash-reporting tests still passed. Adds a pseudo-file seam (shared by both
modules, as the single reader already was) and tests the namespace mount-root
fallback, the root-cgroup rejection, the all-undefined silence and PSI's
absence.
pre-gone-host-memory.test.ts calls getSystemMemoryDetails('linux') with the real
readers live, so on a memory-capped or busy Linux runner its label assertion
reads the runner's own cgroup and fails; seams it like the new files do.
Corrects the doc's blanket "an absent row means could not measure": memory.max
reading `max` is also reported as an absent field, and CgroupCurrentMB is what
separates the two.
This commit is contained in:
@@ -48,7 +48,13 @@ up to 10 s before it (see `pre-gone-host-memory.ts`).
|
||||
| `systemMemoryStall{Some,Full}Avg{10,60}Pct` | `/proc/pressure/memory` |
|
||||
| `systemMemoryCgroupStall{Some,Full}Avg{10,60}Pct` | the cgroup's `memory.pressure` |
|
||||
|
||||
An absent row means "could not measure", never "unlimited" or "calm". cgroup v1
|
||||
An absent row means "could not measure", never "calm" — with one exception to
|
||||
read carefully: `memory.max` and `memory.high` read the literal string `max`
|
||||
when no ceiling is set, and that is reported as an absent `CgroupMaxMB` /
|
||||
`CgroupHighMB`, not as a number. So the ceiling fields alone cannot separate "no
|
||||
ceiling" from "unreadable"; `systemMemoryCgroupCurrentMB` is the tell. Present
|
||||
means the cgroup was read and the missing ceiling really is unlimited; no
|
||||
`Cgroup*` field at all means nothing was measurable. cgroup v1
|
||||
is not read at all: its limit is not resolvable from `/proc/self/cgroup` without
|
||||
mount parsing, and a half-right ceiling is worse than none.
|
||||
|
||||
|
||||
@@ -5,7 +5,9 @@ import {
|
||||
parseCgroupMemoryEvent,
|
||||
parseCgroupV2Path,
|
||||
readLinuxCgroupMemoryLimit,
|
||||
setLinuxCgroupMemoryLimitReaderForTest
|
||||
resolveCgroupV2MemoryDir,
|
||||
setLinuxCgroupMemoryLimitReaderForTest,
|
||||
setLinuxPseudoFileReaderForTest
|
||||
} from './linux-cgroup-memory-limit'
|
||||
import { setLinuxMemoryPressureStallReaderForTest } from './linux-memory-pressure-stall'
|
||||
import { getSystemMemoryDetails, setSystemMemoryInfoReaderForTest } from './system-memory-details'
|
||||
@@ -33,9 +35,87 @@ beforeEach(() => {
|
||||
afterEach(() => {
|
||||
setLinuxCgroupMemoryLimitReaderForTest(null)
|
||||
setLinuxMemoryPressureStallReaderForTest(null)
|
||||
setLinuxPseudoFileReaderForTest(null)
|
||||
setSystemMemoryInfoReaderForTest(null)
|
||||
})
|
||||
|
||||
/** Only the listed paths exist; anything else reads as an unreadable pseudo-file. */
|
||||
function fakeLinuxPseudoFiles(files: Record<string, string>): void {
|
||||
setLinuxPseudoFileReaderForTest((filePath) => files[filePath])
|
||||
}
|
||||
|
||||
const SANDBOX_CGROUP_PATH = '0::/user.slice/user-1000.slice/app.slice/orca.scope\n'
|
||||
|
||||
// Everything below the seam that the reader test double skips: which directory
|
||||
// the sysfs reads actually land in, and whether an unresolvable one stays quiet.
|
||||
describe('cgroup v2 memory directory resolution', () => {
|
||||
it('reads the directory /proc/self/cgroup names when it exists', () => {
|
||||
fakeLinuxPseudoFiles({
|
||||
'/proc/self/cgroup': SANDBOX_CGROUP_PATH,
|
||||
'/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.current': '512\n'
|
||||
})
|
||||
|
||||
expect(resolveCgroupV2MemoryDir()).toBe(
|
||||
'/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope'
|
||||
)
|
||||
})
|
||||
|
||||
it('falls back to the mount root, which is our own cgroup inside a namespace', () => {
|
||||
// The sandbox case the header claims to cover: the reported path is a host
|
||||
// path that does not exist in here, and the mount root IS our cgroup.
|
||||
fakeLinuxPseudoFiles({
|
||||
'/proc/self/cgroup': SANDBOX_CGROUP_PATH,
|
||||
'/sys/fs/cgroup/memory.current': '900000000\n',
|
||||
'/sys/fs/cgroup/memory.max': '1073741824\n'
|
||||
})
|
||||
|
||||
expect(resolveCgroupV2MemoryDir()).toBe('/sys/fs/cgroup')
|
||||
expect(readLinuxCgroupMemoryLimit('linux')).toMatchObject({
|
||||
maxBytes: 1_073_741_824,
|
||||
currentBytes: 900_000_000
|
||||
})
|
||||
})
|
||||
|
||||
it('claims nothing when neither candidate has memory.current', () => {
|
||||
// A v1-only or hybrid host: the unified root exists but carries no memory
|
||||
// controller, and the host root cgroup never has memory.current.
|
||||
fakeLinuxPseudoFiles({ '/proc/self/cgroup': SANDBOX_CGROUP_PATH })
|
||||
|
||||
expect(resolveCgroupV2MemoryDir()).toBeUndefined()
|
||||
expect(readLinuxCgroupMemoryLimit('linux')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('reads the ceiling, the throttle and the events off the resolved directory', () => {
|
||||
fakeLinuxPseudoFiles({
|
||||
'/proc/self/cgroup': SANDBOX_CGROUP_PATH,
|
||||
'/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.current': '4200000000',
|
||||
'/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.max': '4294967296',
|
||||
'/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.high': 'max\n',
|
||||
'/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.events':
|
||||
'low 0\nhigh 7\nmax 2\noom 1\noom_kill 1\n'
|
||||
})
|
||||
|
||||
expect(readLinuxCgroupMemoryLimit('linux')).toEqual({
|
||||
maxBytes: 4_294_967_296,
|
||||
// `max` is no ceiling, and must not surface as one just because it was read.
|
||||
highBytes: undefined,
|
||||
currentBytes: 4_200_000_000,
|
||||
oomKillCount: 1,
|
||||
maxEventCount: 2,
|
||||
highEventCount: 7
|
||||
})
|
||||
})
|
||||
|
||||
it('says nothing rather than a row of undefineds when the files are garbage', () => {
|
||||
fakeLinuxPseudoFiles({
|
||||
'/proc/self/cgroup': SANDBOX_CGROUP_PATH,
|
||||
'/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.current': 'max'
|
||||
})
|
||||
|
||||
expect(readLinuxCgroupMemoryLimit('linux')).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('linux cgroup v2 memory limit', () => {
|
||||
it('takes the unified-hierarchy line, not a v1 controller line', () => {
|
||||
const procSelfCgroup = [
|
||||
|
||||
@@ -38,8 +38,10 @@ export type LinuxCgroupMemoryLimit = {
|
||||
|
||||
type LinuxCgroupMemoryLimitReader = () => LinuxCgroupMemoryLimit | undefined
|
||||
|
||||
type LinuxPseudoFileReader = (filePath: string) => string | undefined
|
||||
|
||||
/** Absent is a normal answer here: hardened hosts, WSL and containers hide these. */
|
||||
export function readLinuxPseudoFile(filePath: string): string | undefined {
|
||||
function readLinuxPseudoFileFromDisk(filePath: string): string | undefined {
|
||||
try {
|
||||
return readFileSync(filePath, 'utf8')
|
||||
} catch {
|
||||
@@ -47,6 +49,17 @@ export function readLinuxPseudoFile(filePath: string): string | undefined {
|
||||
}
|
||||
}
|
||||
|
||||
let pseudoFileReader: LinuxPseudoFileReader = readLinuxPseudoFileFromDisk
|
||||
|
||||
/** The seam the sysfs resolution itself is tested through; PSI reads through it too. */
|
||||
export function setLinuxPseudoFileReaderForTest(next: LinuxPseudoFileReader | null): void {
|
||||
pseudoFileReader = next ?? readLinuxPseudoFileFromDisk
|
||||
}
|
||||
|
||||
export function readLinuxPseudoFile(filePath: string): string | undefined {
|
||||
return pseudoFileReader(filePath)
|
||||
}
|
||||
|
||||
/** The unified-hierarchy line is the one with an empty controller list. */
|
||||
export function parseCgroupV2Path(procSelfCgroup: string): string | undefined {
|
||||
for (const line of procSelfCgroup.split('\n')) {
|
||||
|
||||
@@ -5,7 +5,10 @@ import {
|
||||
readLinuxMemoryPressureStall,
|
||||
setLinuxMemoryPressureStallReaderForTest
|
||||
} from './linux-memory-pressure-stall'
|
||||
import { setLinuxCgroupMemoryLimitReaderForTest } from './linux-cgroup-memory-limit'
|
||||
import {
|
||||
setLinuxCgroupMemoryLimitReaderForTest,
|
||||
setLinuxPseudoFileReaderForTest
|
||||
} from './linux-cgroup-memory-limit'
|
||||
import { getSystemMemoryDetails, setSystemMemoryInfoReaderForTest } from './system-memory-details'
|
||||
import {
|
||||
preGoneSystemMemoryDetails,
|
||||
@@ -36,9 +39,54 @@ beforeEach(() => {
|
||||
afterEach(() => {
|
||||
setLinuxCgroupMemoryLimitReaderForTest(null)
|
||||
setLinuxMemoryPressureStallReaderForTest(null)
|
||||
setLinuxPseudoFileReaderForTest(null)
|
||||
setSystemMemoryInfoReaderForTest(null)
|
||||
})
|
||||
|
||||
/** Only the listed paths exist; anything else reads as an unreadable pseudo-file. */
|
||||
function fakeLinuxPseudoFiles(files: Record<string, string>): void {
|
||||
setLinuxPseudoFileReaderForTest((filePath) => files[filePath])
|
||||
}
|
||||
|
||||
const CALM_PRESSURE = 'some avg10=0.00 avg60=0.00 avg300=0.00 total=0\n'
|
||||
|
||||
// Below the reader seam: which files the procfs reader actually opens, and
|
||||
// whether an absent PSI stays absent instead of reading as a calm host.
|
||||
describe('reading PSI off procfs and the cgroup', () => {
|
||||
it("reads the host file and the resolved cgroup's own memory.pressure", () => {
|
||||
fakeLinuxPseudoFiles({
|
||||
'/proc/self/cgroup': '0::/user.slice/orca.scope\n',
|
||||
'/sys/fs/cgroup/user.slice/orca.scope/memory.current': '512\n',
|
||||
'/proc/pressure/memory': PROC_PRESSURE_MEMORY,
|
||||
'/sys/fs/cgroup/user.slice/orca.scope/memory.pressure':
|
||||
'some avg10=71.20 avg60=60.00 avg300=20.00 total=1\nfull avg10=58.90 avg60=41.30 avg300=9.00 total=2\n'
|
||||
})
|
||||
|
||||
expect(readLinuxMemoryPressureStall('linux')).toEqual({
|
||||
host: { someAvg10: 61.4, someAvg60: 48.22, fullAvg10: 44.1, fullAvg60: 30.05 },
|
||||
cgroup: { someAvg10: 71.2, someAvg60: 60, fullAvg10: 58.9, fullAvg60: 41.3 }
|
||||
})
|
||||
})
|
||||
|
||||
it('still reports the host file when the cgroup cannot be resolved', () => {
|
||||
fakeLinuxPseudoFiles({ '/proc/pressure/memory': CALM_PRESSURE })
|
||||
|
||||
expect(readLinuxMemoryPressureStall('linux')).toEqual({
|
||||
host: { someAvg10: 0, someAvg60: 0 },
|
||||
cgroup: undefined
|
||||
})
|
||||
})
|
||||
|
||||
it('stays silent on a kernel built without CONFIG_PSI', () => {
|
||||
fakeLinuxPseudoFiles({
|
||||
'/proc/self/cgroup': '0::/user.slice/orca.scope\n',
|
||||
'/sys/fs/cgroup/user.slice/orca.scope/memory.current': '512\n'
|
||||
})
|
||||
|
||||
expect(readLinuxMemoryPressureStall('linux')).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('linux PSI memory stall', () => {
|
||||
it('takes avg10 and avg60 off both the some and full lines', () => {
|
||||
expect(parseMemoryPressureStall(PROC_PRESSURE_MEMORY)).toEqual({
|
||||
|
||||
@@ -10,6 +10,8 @@ import {
|
||||
type SwapVolumeFreeSpace
|
||||
} from './swap-volume-free-space'
|
||||
import { samplePreGoneSystemMemory } from './pre-gone-host-memory'
|
||||
import { setLinuxCgroupMemoryLimitReaderForTest } from './linux-cgroup-memory-limit'
|
||||
import { setLinuxMemoryPressureStallReaderForTest } from './linux-memory-pressure-stall'
|
||||
import {
|
||||
buildProcessGoneCrashDetails,
|
||||
resetPreGoneCrashSamplingForTest,
|
||||
@@ -82,6 +84,10 @@ describe('pre-gone host memory', () => {
|
||||
resetPreGoneCrashSamplingForTest()
|
||||
setSystemMemoryInfoReaderForTest(null)
|
||||
setSwapVolumeFreeSpaceReaderForTest(null)
|
||||
// Without these the linux readings below come from the CI host's own cgroup
|
||||
// and PSI, so a capped or busy runner rewrites the label under test.
|
||||
setLinuxCgroupMemoryLimitReaderForTest(() => undefined)
|
||||
setLinuxMemoryPressureStallReaderForTest(() => undefined)
|
||||
appMetricsMock.mockClear()
|
||||
appMetricsMock.mockReturnValue(BROWSER_AND_RENDERER)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user