From b79bcbd04e50b86186645bcba04d811671eb3035 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Wed, 15 Jul 2026 21:40:51 -0400 Subject: [PATCH] feat: add disconnected relay cache eviction Co-authored-by: Orca --- .../workflows/ssh-relay-runtime-artifacts.yml | 4 + .../ssh-relay-runtime-workflow.test.mjs | 2 + ...elease-implementation-checklist-summary.md | 15 +- ...github-release-implementation-checklist.md | 184 ++++++++++- ...h-relay-artifact-cache-entry-accounting.ts | 67 ++++ ...lay-artifact-cache-entry-full-size.test.ts | 63 +++- ...h-relay-artifact-cache-eviction-fixture.ts | 45 +++ .../ssh-relay-artifact-cache-eviction.test.ts | 273 ++++++++++++++++ .../ssh/ssh-relay-artifact-cache-eviction.ts | 293 ++++++++++++++++++ ...-relay-artifact-cache-in-use-inspection.ts | 87 ++++++ ...-relay-artifact-cache-in-use-lease.test.ts | 178 +++++++++++ .../ssh-relay-artifact-cache-in-use-lease.ts | 258 +++++++++++++++ .../ssh-relay-artifact-cache-in-use-record.ts | 113 +++++++ .../ssh/ssh-relay-artifact-cache-recency.ts | 113 +++++++ 14 files changed, 1676 insertions(+), 19 deletions(-) create mode 100644 src/main/ssh/ssh-relay-artifact-cache-entry-accounting.ts create mode 100644 src/main/ssh/ssh-relay-artifact-cache-eviction-fixture.ts create mode 100644 src/main/ssh/ssh-relay-artifact-cache-eviction.test.ts create mode 100644 src/main/ssh/ssh-relay-artifact-cache-eviction.ts create mode 100644 src/main/ssh/ssh-relay-artifact-cache-in-use-inspection.ts create mode 100644 src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts create mode 100644 src/main/ssh/ssh-relay-artifact-cache-in-use-lease.ts create mode 100644 src/main/ssh/ssh-relay-artifact-cache-in-use-record.ts create mode 100644 src/main/ssh/ssh-relay-artifact-cache-recency.ts diff --git a/.github/workflows/ssh-relay-runtime-artifacts.yml b/.github/workflows/ssh-relay-runtime-artifacts.yml index 4d8f5b7de5c..289adc424d3 100644 --- a/.github/workflows/ssh-relay-runtime-artifacts.yml +++ b/.github/workflows/ssh-relay-runtime-artifacts.yml @@ -266,6 +266,8 @@ jobs: src/main/ssh/ssh-relay-artifact-extraction.test.ts \ src/main/ssh/ssh-relay-artifact-cache-lock.test.ts \ src/main/ssh/ssh-relay-artifact-cache-entry.test.ts \ + src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts \ + src/main/ssh/ssh-relay-artifact-cache-eviction.test.ts \ src/main/ssh/ssh-relay-runtime-identity.test.ts \ config/scripts/ssh-relay-runtime-compatibility.test.mjs \ config/scripts/ssh-relay-node-release-verification.test.mjs \ @@ -798,6 +800,8 @@ jobs: src/main/ssh/ssh-relay-artifact-extraction.test.ts ` src/main/ssh/ssh-relay-artifact-cache-lock.test.ts ` src/main/ssh/ssh-relay-artifact-cache-entry.test.ts ` + src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts ` + src/main/ssh/ssh-relay-artifact-cache-eviction.test.ts ` src/main/ssh/ssh-relay-runtime-identity.test.ts ` config/scripts/ssh-relay-runtime-compatibility.test.mjs ` config/scripts/ssh-relay-node-release-verification.test.mjs ` diff --git a/config/scripts/ssh-relay-runtime-workflow.test.mjs b/config/scripts/ssh-relay-runtime-workflow.test.mjs index de35c44d31b..0e9b36bf790 100644 --- a/config/scripts/ssh-relay-runtime-workflow.test.mjs +++ b/config/scripts/ssh-relay-runtime-workflow.test.mjs @@ -124,6 +124,8 @@ describe('SSH relay runtime artifact workflow', () => { 'ssh-relay-artifact-extraction.test.ts', 'ssh-relay-artifact-cache-lock.test.ts', 'ssh-relay-artifact-cache-entry.test.ts', + 'ssh-relay-artifact-cache-in-use-lease.test.ts', + 'ssh-relay-artifact-cache-eviction.test.ts', 'ssh-relay-runtime-identity.test.ts' ]) { // Why: portable desktop selection contracts need proof on every native runner family, not diff --git a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md index 8940f7da7fe..117702c0387 100644 --- a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md +++ b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md @@ -9,8 +9,12 @@ keeps commands, hashes, runner identities, timings, and failure details. A checked box means the work has evidence in the detailed ledger. Design approval alone does not complete a box. -Active checkpoint: **Milestone 5 / Work Package 4 desktop cache boundary — immutable cache entry -closed; in-use lease/2 GiB eviction is next, 2026-07-15, Codex implementation owner.** Exact-head +Active checkpoint: **Milestone 5 / Work Package 4 desktop cache boundary — local lease/recency/ +eviction contracts are green; wiring full-size all-six native proof, 2026-07-15, Codex implementation +owner.** Fourteen focused contracts, workflow wiring, broad relay/release suites, typecheck, lint, and +static gates pass locally under `E-M5-ARTIFACT-CACHE-EVICTION-LOCAL-001` and +`E-M5-ARTIFACT-CACHE-EVICTION-CI-WIRING-LOCAL-001`; the package remains open until exact-head native +CI supplies all-six full-size retention/eviction metrics. Exact-head run [29462394311](https://github.com/stablyai/orca/actions/runs/29462394311) passes all 17 cache-entry contracts and exact full-size cold/warm measurements on Linux, macOS, and Windows, x64 and arm64, under `E-M5-ARTIFACT-CACHE-ENTRY-CI-001`. Cold publication is 1.17–6.17s and 34.5–48.8 MiB @@ -18,7 +22,7 @@ incremental RSS; warm verified lookup is 0.12–0.70s and 0–6.9 MiB. PR Checks [29462394310](https://github.com/stablyai/orca/actions/runs/29462394310) and Golden E2E [29462394344](https://github.com/stablyai/orca/actions/runs/29462394344) pass. Windows arm64 retains only the declared hosted build-26200 versus required-26100 floor gap after its primary runtime/cache -proof. Desktop consumers, eviction, SSH transfer/install, mode wiring, tuple enablement, publication, +proof. Desktop consumers, SSH transfer/install, mode wiring, tuple enablement, publication, production keys/environment/seed, and merge to `main` remain disconnected. ## Safety status @@ -317,8 +321,9 @@ production keys/environment/seed, and merge to `main` remain disconnected. - [ ] Add exclusive staging, atomic publication, quarantine, locking, and the 2 GiB cache policy. Locking and immutable publication/lookup/quarantine are closed locally and on all six native clients under `E-M5-ARTIFACT-CACHE-LOCK-CI-001` and - `E-M5-ARTIFACT-CACHE-ENTRY-CI-001`. In-use leases, recency, and bounded 2 GiB eviction remain - the only open part of this combined item. + `E-M5-ARTIFACT-CACHE-ENTRY-CI-001`. In-use leases, recency, exact byte accounting, and bounded + eviction pass locally under `E-M5-ARTIFACT-CACHE-EVICTION-LOCAL-001`; all-six native/full-size + eviction evidence remains the only open part of this combined item. - [ ] Prove verified cached bytes can be transferred while the client is offline. - [ ] Preserve `ORCA_RELAY_PATH` behind the official-build trust boundary. diff --git a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md index 6aa5897f6e1..2bded9c897a 100644 --- a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md +++ b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md @@ -8,8 +8,8 @@ work; keep exact commands, runner identities, hashes, metrics, and residual gaps Date created: 2026-07-14
Last updated: 2026-07-15
-Current phase: Milestone 5 / Work Package 4 desktop cache boundary — **Checkpoint — 2026-07-15, Codex implementation owner: immutable cache-entry gate closed; evidence checkpoint/push required before in-use lease/eviction work**. E-M5-ARTIFACT-CACHE-ENTRY-CI-001 closes all six primary Node 24 source and exact full-size cold-publication/warm-verified-lookup cells at `3c1ead8fb1e1f3b64abaf78705abb0c801a1d64b`. PR Checks and Golden E2E pass. No cache consumer, eviction, SSH path, mode, tuple, release write, or default behavior is connected.
-Session checkpoint: **In progress — 2026-07-15, Codex implementation owner** — resumed with user authority for end-to-end PR-contained implementation, commits, pushes, CI, rehearsals, and release writes, with merging to `main` explicitly prohibited. The first native run exposed and recorded Windows open-handle rename and equivalent zlib-error-oracle REDs; narrow corrections passed locally and at exact-head CI. In replacement run `29459424864`, all six primary jobs pass the 10 lock and 21 extraction contracts plus deterministic artifact build/smoke. The workflow's sole failure is the retained Windows arm64 hosted build-26200 versus required-26100 floor rejection after runtime/archive/tree/Node/PTTY/watcher proof; no tuple is enabled from it. PR Checks `29459424977` and Golden E2E `29459424789` pass. There is no release publication, cache entry, cache consumer, SSH transfer/install, tuple enablement, or production/default behavior change. Legacy remains the production default.
+Current phase: Milestone 5 / Work Package 4 desktop cache boundary — **In progress — 2026-07-15, Codex implementation owner: wire and prove disconnected in-use lease/recency/2 GiB eviction on all six native runners**. The purpose-named RED is recorded, 14 focused contracts and the 126-test artifact source suite are locally green, and the next gate is full-size eviction measurement plus workflow-contract and all-six native evidence. Keep every Electron, downloader, SSH, setting, tuple, publication, and default-path consumer disconnected.
+Session checkpoint: **In progress — 2026-07-15, Codex implementation owner** — user authority covers end-to-end PR-contained implementation, commits, pushes, CI, rehearsals, and release writes; merging to `main` remains explicitly prohibited. The active uncommitted package implements only cross-process in-use leases, persistent recency, exact logical-byte accounting, and bounded eviction. It has no Electron, downloader, SSH, setting, tuple, publication, or default-path consumer. Legacy remains the production default.
Primary design: [SSH relay GitHub Release plan](./2026-07-14-ssh-relay-github-release-plan.html)
Motivating issues: [#8450](https://github.com/stablyai/orca/issues/8450), [#1693](https://github.com/stablyai/orca/issues/1693) @@ -13490,6 +13490,170 @@ src/main/ssh/ssh-relay-artifact-cache-entry-full-size.test.ts`: workflow and foc - Follow-up: checkpoint and push this evidence, then audit and RED-test only the disconnected in-use lease/recency/2 GiB eviction package. Keep consumers and SSH behavior separate. +### E-M5-ARTIFACT-CACHE-EVICTION-AUDIT-001 — In-use lease, recency, and 2 GiB eviction boundary + +- Date: 2026-07-15 +- Owner: Codex implementation owner +- Source: exact pushed cache-entry evidence head `0ae3e005f` on draft PR #8741. +- Exact audit commands: complete reads of the cache entry, proof, verification, content-lock record, + content-lock lease, and content-lock tests; `rg --files src/main/ssh` over relay cache/artifact + modules; targeted `rg -n` over cache eviction, in-use lease, recency/LRU, protected content, and + native workflow invocations in `src/main`, `src/shared`, `.github/workflows`, and `config/scripts`. +- Findings: + - no relay cache eviction, in-use lease, persistent recency, exact cache-byte accounting, or + consumer exists. The immutable entry becomes selectable only after complete verification, but a + future consumer needs a cross-process reference before the content lock is released to eviction; + - in-use references must be shared, not exclusive. Each exact content ID therefore owns a + token-named lease directory with a strict nonce/host/PID/acquired/heartbeat record and owner-handle + heartbeat. Active, malformed, remote-host, PID-reused, or otherwise ambiguous leases protect the + entry. Only a confirmed stale same-host dead owner may be tombstoned and removed; + - lease acquisition and eviction both acquire the proven per-content lock. Acquisition rechecks the + exact final entry path before making its lease visible; eviction rechecks entry state, staging, and + leases while holding the same lock. This closes the lookup-to-reference/delete race without + changing cache lookup or adding a consumer; + - successful lease acquisition/release records persistent recency outside the immutable entry via + token-shaped append records. Readers take the greatest valid timestamp; malformed or unexpected + recency is ambiguous and conservatively protects the entry. Missing recency uses the immutable + entry timestamp as its initial publication order; + - one global eviction transaction is serialized by the proven lock implementation in a dedicated + eviction namespace, so no valid content digest is reserved. Per-entry locks still serialize with + publication and reference acquisition; + - exact accounting sums stable logical regular-file sizes without following links and rejects + unsafe-integer arithmetic or ambiguous/special trees. The 2 GiB limit applies to complete + selectable entries; staging, locked, ambiguous, current-manifest, and running content are never + deleted and can leave an explicit over-budget residual; + - eviction orders idle complete entries by persistent LRU with a content-ID tie break. Strictly + protected content is never removed. Caller-declared current/previous retention is considered only + after ordinary candidates and may be removed only when required to meet the cap, matching “retain + when they fit”; + - deletion rechecks all guards under the content lock, atomically renames only the exact final entry + to a token-shaped unselectable tombstone, then removes owned bytes. Cancellation is checked before + each bounded scan/acquire/delete stage and all acquired locks/leases must settle. +- Required purpose-named contracts: exact paths/limits; multiple concurrent references; real + cross-process visibility; heartbeat and owner-only release; stale-dead cleanup; live/malformed/ + remote/displaced-owner preservation; exact logical-byte accounting; deterministic LRU and tie + break; hard and preferred retention; staging/content-lock protection; concurrent eviction; + cancellation settlement; symlink/special/unsafe state protection; and no partial/tombstone selection. +- Deliberately separate: cache-root selection, embedded-manifest loading, proxy/certificates, + downloader calls, offline transfer, SSH, remote install, ORCA_RELAY_PATH, fallback classification, + settings/mode, tuple enablement, release publication, and production/default behavior. +- Follow-up: add `ssh-relay-artifact-cache-eviction.test.ts` with production modules absent, record + the RED, then implement only the audited lease/recency/eviction modules and require focused, + broad/static, native-workflow wiring, and all-six native evidence. + +### E-M5-ARTIFACT-CACHE-EVICTION-LOCAL-RED-001 — In-use lease and eviction capabilities are absent + +- Date: 2026-07-15 +- Owner: Codex implementation owner +- Source: purpose-named `ssh-relay-artifact-cache-in-use-lease.test.ts` and + `ssh-relay-artifact-cache-eviction.test.ts` atop pushed evidence head `0ae3e005f`; production + lease/eviction modules deliberately absent. +- Exact command: `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts +--maxWorkers=1 src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts +src/main/ssh/ssh-relay-artifact-cache-eviction.test.ts`. +- Environment: macOS 26.2 arm64 build 25C56, Node v26.0.0, pnpm 10.24.0, Vitest 4.1.5. +- Result: required RED; two failed suites / zero collected tests in 642ms Vitest / 2.57s wall, + 132,841,472-byte maximum RSS, zero swaps. Imports fail because both audited production modules do + not exist. +- Contracts encoded before implementation: exact paths/limits; shared concurrent and real + second-process leases; heartbeat and displaced-owner release; missing-entry rejection; exact + logical-byte accounting; LRU; hard/preferred retention; active/malformed/live/remote/stale-dead + lease handling; staging/content-lock/cancellation safety; concurrent evictors; and unsafe-link + preservation. +- Consumer-disconnection oracle: tests import the new source capabilities directly and construct + isolated cache fixtures. No Electron, downloader, SSH, setting, mode, tuple, release, or default + call site is added. +- Follow-up: implement only concrete in-use record, recency, lease, and eviction responsibilities; + make both purpose suites green without weakening conservative ambiguity or reference safety. + +### E-M5-ARTIFACT-CACHE-EVICTION-LOCAL-001 — Disconnected in-use leases and bounded eviction pass locally + +- Date: 2026-07-15 +- Owner: Codex implementation owner +- Source: uncommitted package atop exact pushed evidence head `0ae3e005f`, following + E-M5-ARTIFACT-CACHE-EVICTION-AUDIT-001 and E-M5-ARTIFACT-CACHE-EVICTION-LOCAL-RED-001. +- Implementation: + - strict token-named shared lease records bind content ID, nonce, host, PID, acquisition, heartbeat, + and directory identity. Acquisition and release serialize through the proven per-content lock; + - 5s heartbeats and owner-only release preserve active and displaced owners. Active, malformed, + remote-host, PID-reused/live, or otherwise ambiguous references block deletion. Only a stale + same-host owner whose PID is confirmed dead is atomically tombstoned and reclaimed; + - persistent token-shaped recency records live outside immutable entries and are bounded to the + newest valid timestamp. Missing recency starts at publication mtime; malformed/racing state is + ambiguous and protected; + - exact stable logical-file accounting follows no links and bounds each entry to 10,000 members. + Unsafe, special, unstable, or unaccountable trees remain protected; + - one dedicated global eviction lock serializes transactions, while per-content locks close + publication/reference/delete races. The default cap is 2 GiB; hard-protected content is never + removed, preferred current/previous content follows ordinary LRU candidates, and complete idle + entries become unselectable by atomic tombstone rename before owned recursive deletion. +- Exact focused command: `/usr/bin/time -l pnpm exec vitest run --config +config/vitest.config.ts --maxWorkers=1 +src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts +src/main/ssh/ssh-relay-artifact-cache-eviction.test.ts`. +- Final focused result: two passed files / 14 passed tests in 2.10s Vitest / 4.12s wall, + 131,629,056-byte maximum RSS, zero swaps on macOS 26.2 arm64 build 25C56, Node v26.0.0, pnpm + 10.24.0, Vitest 4.1.5. +- Contracts prove exact paths/limits and invalid-input rejection, multiple references, real + child-process visibility, heartbeat/displaced-owner safety, missing-entry rejection, bounded + recency, exact accounting/deterministic LRU, hard/preferred retention, active/malformed/live/ + remote/stale-dead behavior, staging/content-lock/cancellation settlement, concurrent eviction, and + unsafe linked-tree preservation. +- Exact broader commands/results: + - `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1 +src/main/ssh/ssh-relay-artifact-*.test.ts`: eight passed files / two environment-skipped files, + 126 passed / two skipped tests in 15.49s Vitest / 23.55s wall, 159,383,552-byte maximum RSS, + zero swaps; + - `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1 +src/main/ssh/ssh-relay-*.test.ts`: 21 passed files / three skipped files, 294 passed / three + skipped tests in 52.31s Vitest / 55.78s wall, 205,340,672-byte maximum RSS, zero swaps; + - `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1 +config/scripts/ssh-relay-runtime-*.test.mjs`: 50 passed files / 279 passed tests in 86.52s + Vitest / 93.38s wall, 188,940,288-byte maximum RSS, zero swaps; + - `/usr/bin/time -l pnpm typecheck`: pass in 5.70s wall with 1,208,139,776-byte maximum RSS; + - `/usr/bin/time -l pnpm lint`: pass in 30.71s wall with 2,093,400,064-byte maximum RSS. Full + lint retains only existing unrelated warnings and includes switch exhaustiveness, 41 reliability + gates, max-lines ratchet, bundled-skill, and localization gates; + - `pnpm exec oxfmt --write` over all package/workflow/checklist files: pass in 4.334s; + `/usr/bin/git diff --check`: pass. No max-lines suppression or ratchet exception was added. +- Consumer/diff oracle: direct source tests are the only caller. The preserved Milestone 0 resolver + pair has zero diff. No Electron/cache-root/downloader/SSH/setting/mode/tuple/publication/default + consumer exists; legacy remains the only production path. +- Does not prove: Node 24/native-client behavior, exact full-size active retention/eviction resources, + disk-full/read-only/quota/inode faults, crash/power-loss durability, packaged Electron, SSH, native + trust, or any enabled tuple. All remain separately gated. +- Follow-up: wire the purpose suites into both native families and extend the exact full-size cache + measurement through active retention, release, exact-byte eviction, and cleanup. Push the isolated + package and require all-six native source/full-size evidence before closing this combined cache + item or adding any consumer. + +### E-M5-ARTIFACT-CACHE-EVICTION-CI-WIRING-LOCAL-001 — Both native families require source and full-size eviction gates + +- Date: 2026-07-15 +- Owner: Codex implementation owner +- Source: uncommitted workflow/test extension atop E-M5-ARTIFACT-CACHE-EVICTION-LOCAL-001. +- Implementation: both POSIX and PowerShell native Node 24 source commands now require the 14 + purpose-named lease/eviction contracts. The existing exact-artifact cache measurement keeps every + cold/warm JSON field and additionally acquires a real in-use lease, proves a zero-byte-cap eviction + retains the complete entry, releases the lease, then measures exact logical bytes reclaimed, + elapsed time, incremental RSS, entry removal, and test-root cleanup. +- Budgets/oracles: each retention and eviction transaction is bounded at 5m and 80 MiB incremental + RSS; lease acquisition is bounded at 2m. The test timeout is the explicit sum of two entry, two + eviction, one acquisition, and 10s cleanup budgets. The native workflow must remove the measurement + root after the test on both shell families. +- Exact command: `/usr/bin/time -l pnpm exec vitest run --config config/vitest.config.ts +--maxWorkers=1 config/scripts/ssh-relay-runtime-workflow.test.mjs +src/main/ssh/ssh-relay-artifact-cache-entry-full-size.test.ts`. +- Result without full-size environment: workflow file passes seven tests and the full-size file skips + its one environment-gated test as intended in 1.36s Vitest / 3.41s wall, 134,496,256-byte maximum + RSS, zero swaps. The standalone workflow command also passed seven tests in 861ms Vitest / 4.32s + wall, 131,629,056-byte maximum RSS. +- Does not prove: actual full-size eviction or native runner behavior. Those require the pushed exact + head on Linux, macOS, and Windows x64/arm64; no checklist box closes from this local wiring proof. +- Follow-up: commit and push this isolated package, then capture exact run/job IDs, commit SHA, + runner images/architectures, per-tuple retention/eviction timings and RSS, exact reclaimed bytes, + cleanup results, and residual floor gaps under a new CI evidence ID. + ## Accepted Gaps No product gap is accepted merely because it appears in this list. Each entry requires explicit @@ -13548,15 +13712,13 @@ The project is not complete until every applicable item below is checked with ev ## Next Required Action -Audit cache ownership across processes/windows, immutable entry naming/proofs, exclusive locking, -stale-lock recovery, atomic publication, corruption quarantine, in-use protection, and bounded 2 GiB -eviction. Record the audit and add purpose-named failing contracts before implementing this narrow, -disconnected cache-state package. Do not add a desktop call site, SSH transfer/install, mode wiring, -tuple enablement, release publication, or default behavior. Keep Node upstream `.tar.xz` inputs, -Windows ZIP, -`ORCA_RELAY_PATH`, existing desktop required-assets behavior, detached-signature byte encoding, -Windows arm64 build 26100, macOS 13.5, Linux kernel 4.18, release-cut, desktop builds, publication, -and every tuple separately gated. +Commit and push the disconnected in-use lease/recency/2 GiB eviction package, then require both +purpose suites and exact full-size active-retention/release/eviction measurements on all six native +jobs. Do not add a desktop call site, SSH transfer/install, mode wiring, tuple enablement, release +publication, or default behavior until that CI evidence is complete. +Keep Node upstream `.tar.xz` inputs, Windows ZIP, `ORCA_RELAY_PATH`, existing desktop required-assets +behavior, detached-signature byte encoding, Windows arm64 build 26100, macOS 13.5, Linux kernel 4.18, +release-cut, desktop builds, publication, and every tuple separately gated. Cross-family Layer B targets, the protected manifest-signing environment, oldest-baseline/native- trust cells, and the paired legacy performance baseline remain release/default-path blockers. No diff --git a/src/main/ssh/ssh-relay-artifact-cache-entry-accounting.ts b/src/main/ssh/ssh-relay-artifact-cache-entry-accounting.ts new file mode 100644 index 00000000000..5bed0bb274b --- /dev/null +++ b/src/main/ssh/ssh-relay-artifact-cache-entry-accounting.ts @@ -0,0 +1,67 @@ +import { lstat, readdir } from 'node:fs/promises' +import { join } from 'node:path' + +export const SSH_RELAY_ARTIFACT_CACHE_MAXIMUM_MEMBERS_PER_ENTRY = 10_000 + +function safeAdd(left: number, right: bigint): number | null { + if (right < 0n || right > BigInt(Number.MAX_SAFE_INTEGER)) { + return null + } + const total = left + Number(right) + return Number.isSafeInteger(total) ? total : null +} + +export async function measureSshRelayArtifactCacheEntryLogicalBytes( + root: string, + signal: AbortSignal +): Promise { + const pending = [root] + let members = 0 + let bytes = 0 + while (pending.length > 0) { + signal.throwIfAborted() + if (++members > SSH_RELAY_ARTIFACT_CACHE_MAXIMUM_MEMBERS_PER_ENTRY) { + return null + } + const path = pending.pop()! + const before = await lstat(path, { bigint: true }).catch(() => null) + if (!before || before.isSymbolicLink()) { + return null + } + if (before.isFile()) { + const next = safeAdd(bytes, before.size) + if (next === null) { + return null + } + bytes = next + const after = await lstat(path, { bigint: true }).catch(() => null) + if ( + !after?.isFile() || + after.dev !== before.dev || + after.ino !== before.ino || + after.size !== before.size || + after.mtimeNs !== before.mtimeNs || + after.ctimeNs !== before.ctimeNs + ) { + return null + } + continue + } + if (!before.isDirectory()) { + return null + } + const names = await readdir(path) + pending.push(...names.map((name) => join(path, name))) + const after = await lstat(path, { bigint: true }).catch(() => null) + if ( + !after?.isDirectory() || + after.dev !== before.dev || + after.ino !== before.ino || + after.mtimeNs !== before.mtimeNs || + after.ctimeNs !== before.ctimeNs + ) { + return null + } + } + return bytes +} diff --git a/src/main/ssh/ssh-relay-artifact-cache-entry-full-size.test.ts b/src/main/ssh/ssh-relay-artifact-cache-entry-full-size.test.ts index 1b90768526d..f50f6e88b53 100644 --- a/src/main/ssh/ssh-relay-artifact-cache-entry-full-size.test.ts +++ b/src/main/ssh/ssh-relay-artifact-cache-entry-full-size.test.ts @@ -8,6 +8,14 @@ import { publishSshRelayArtifactCacheEntry, SSH_RELAY_ARTIFACT_CACHE_ENTRY_LIMITS } from './ssh-relay-artifact-cache-entry' +import { + evictSshRelayArtifactCache, + SSH_RELAY_ARTIFACT_CACHE_EVICTION_LIMITS +} from './ssh-relay-artifact-cache-eviction' +import { + acquireSshRelayArtifactCacheInUseLease, + SSH_RELAY_ARTIFACT_CACHE_IN_USE_LIMITS +} from './ssh-relay-artifact-cache-in-use-lease' import type { SshRelaySelectedArtifact } from './ssh-relay-artifact-selector' type MeasurementIdentity = { @@ -87,7 +95,7 @@ async function measure(operation: () => Promise): Promise<{ describe.skipIf(!hasMeasurementInput)('SSH relay full-size immutable artifact cache entry', () => { it( - 'keeps cold publication and verified lookup within desktop resource budgets', + 'keeps cold publication, verified lookup, active retention, and eviction within budgets', async () => { const identity = measurementIdentity( JSON.parse(await readFile(identityPath as string, 'utf8')) as unknown @@ -105,6 +113,19 @@ describe.skipIf(!hasMeasurementInput)('SSH relay full-size immutable artifact ca const warm = await measure(() => lookupSshRelayArtifactCacheEntry({ cacheRoot: cacheRoot as string, artifact }) ) + if (warm.result.kind !== 'hit') { + throw new Error('Full-size cache entry unexpectedly missed after cold publication') + } + const lease = await acquireSshRelayArtifactCacheInUseLease({ + cacheRoot: cacheRoot as string, + entry: warm.result.entry + }) + const retained = await measure(() => + evictSshRelayArtifactCache({ cacheRoot: cacheRoot as string, maximumBytes: 0 }) + ).finally(() => lease.release()) + const eviction = await measure(() => + evictSshRelayArtifactCache({ cacheRoot: cacheRoot as string, maximumBytes: 0 }) + ) console.log( `ssh_relay_full_size_cache=${JSON.stringify({ tupleId: identity.tupleId, @@ -114,7 +135,13 @@ describe.skipIf(!hasMeasurementInput)('SSH relay full-size immutable artifact ca coldElapsedMs: cold.elapsedMs, coldIncrementalRssBytes: cold.incrementalRssBytes, warmElapsedMs: warm.elapsedMs, - warmIncrementalRssBytes: warm.incrementalRssBytes + warmIncrementalRssBytes: warm.incrementalRssBytes, + retainedElapsedMs: retained.elapsedMs, + retainedIncrementalRssBytes: retained.incrementalRssBytes, + evictionElapsedMs: eviction.elapsedMs, + evictionIncrementalRssBytes: eviction.incrementalRssBytes, + evictionInitialBytes: eviction.result.initialBytes, + evictionReclaimedBytes: eviction.result.reclaimedBytes })}` ) expect(cold.result).toMatchObject({ @@ -124,6 +151,25 @@ describe.skipIf(!hasMeasurementInput)('SSH relay full-size immutable artifact ca expandedBytes: identity.archive.expandedSize }) expect(warm.result).toEqual({ kind: 'hit', entry: cold.result }) + expect(retained.result).toMatchObject({ + initialBytes: expect.any(Number), + finalBytes: expect.any(Number), + reclaimedBytes: 0, + evictedContentIds: [], + blockedContentIds: [identity.contentId], + accountingComplete: true + }) + expect(retained.result.initialBytes).toBeGreaterThan(0) + expect(retained.result.finalBytes).toBe(retained.result.initialBytes) + expect(eviction.result).toEqual({ + initialBytes: retained.result.initialBytes, + finalBytes: 0, + reclaimedBytes: retained.result.initialBytes, + evictedContentIds: [identity.contentId], + blockedContentIds: [], + accountingComplete: true + }) + await expect(stat(cold.result.entryPath)).rejects.toMatchObject({ code: 'ENOENT' }) for (const measurement of [cold, warm]) { expect(measurement.elapsedMs).toBeLessThanOrEqual( SSH_RELAY_ARTIFACT_CACHE_ENTRY_LIMITS.transactionTimeoutMs @@ -132,10 +178,21 @@ describe.skipIf(!hasMeasurementInput)('SSH relay full-size immutable artifact ca SSH_RELAY_ARTIFACT_CACHE_ENTRY_LIMITS.maximumIncrementalMemoryBytes ) } + for (const measurement of [retained, eviction]) { + expect(measurement.elapsedMs).toBeLessThanOrEqual( + SSH_RELAY_ARTIFACT_CACHE_EVICTION_LIMITS.transactionTimeoutMs + ) + expect(measurement.incrementalRssBytes).toBeLessThanOrEqual( + SSH_RELAY_ARTIFACT_CACHE_ENTRY_LIMITS.maximumIncrementalMemoryBytes + ) + } } finally { await rm(cacheRoot as string, { recursive: true, force: true }) } }, - SSH_RELAY_ARTIFACT_CACHE_ENTRY_LIMITS.transactionTimeoutMs * 2 + 10_000 + SSH_RELAY_ARTIFACT_CACHE_ENTRY_LIMITS.transactionTimeoutMs * 2 + + SSH_RELAY_ARTIFACT_CACHE_EVICTION_LIMITS.transactionTimeoutMs * 2 + + SSH_RELAY_ARTIFACT_CACHE_IN_USE_LIMITS.acquisitionTimeoutMs + + 10_000 ) }) diff --git a/src/main/ssh/ssh-relay-artifact-cache-eviction-fixture.ts b/src/main/ssh/ssh-relay-artifact-cache-eviction-fixture.ts new file mode 100644 index 00000000000..f5aa49ed94f --- /dev/null +++ b/src/main/ssh/ssh-relay-artifact-cache-eviction-fixture.ts @@ -0,0 +1,45 @@ +import { mkdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' + +import type { SshRelayArtifactCacheEntry } from './ssh-relay-artifact-cache-entry-verification' +import type { SshRelayDigest } from './ssh-relay-runtime-identity' + +export function sshRelayArtifactCacheEvictionContentId(character: string): SshRelayDigest { + return `sha256:${character.repeat(64)}` as SshRelayDigest +} + +export async function createSshRelayArtifactCacheEvictionFixture({ + cacheRoot, + character, + payloadBytes = 32 +}: { + cacheRoot: string + character: string + payloadBytes?: number +}): Promise<{ entry: SshRelayArtifactCacheEntry; logicalBytes: number }> { + const contentId = sshRelayArtifactCacheEvictionContentId(character) + const entryPath = join(cacheRoot, 'entries', contentId.slice('sha256:'.length)) + const runtimeRoot = join(entryPath, 'runtime') + const archivePath = join(entryPath, 'runtime.tar.br') + const proofPath = join(entryPath, 'proof.json') + const archive = Buffer.alloc(payloadBytes, character) + const runtime = Buffer.alloc(payloadBytes + 1, character) + const proof = Buffer.from('{}\n') + await mkdir(runtimeRoot, { recursive: true, mode: 0o700 }) + await writeFile(archivePath, archive, { mode: 0o600 }) + await writeFile(join(runtimeRoot, 'relay.js'), runtime, { mode: 0o700 }) + await writeFile(proofPath, proof, { mode: 0o600 }) + return { + entry: { + contentId, + tupleId: 'linux-x64-glibc', + entryPath, + archivePath, + runtimeRoot, + proofPath, + files: 1, + expandedBytes: runtime.length + }, + logicalBytes: archive.length + runtime.length + proof.length + } +} diff --git a/src/main/ssh/ssh-relay-artifact-cache-eviction.test.ts b/src/main/ssh/ssh-relay-artifact-cache-eviction.test.ts new file mode 100644 index 00000000000..fdaa5467654 --- /dev/null +++ b/src/main/ssh/ssh-relay-artifact-cache-eviction.test.ts @@ -0,0 +1,273 @@ +import { mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { hostname, tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { createSshRelayArtifactCacheEvictionFixture } from './ssh-relay-artifact-cache-eviction-fixture' +import { + evictSshRelayArtifactCache, + SSH_RELAY_ARTIFACT_CACHE_EVICTION_LIMITS +} from './ssh-relay-artifact-cache-eviction' +import { acquireSshRelayArtifactCacheInUseLease } from './ssh-relay-artifact-cache-in-use-lease' +import { acquireSshRelayArtifactCacheLock } from './ssh-relay-artifact-cache-lock' +import type { SshRelayDigest } from './ssh-relay-runtime-identity' + +const temporaryDirectories: string[] = [] + +async function cacheRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-relay-cache-eviction-')) + temporaryDirectories.push(root) + return join(root, 'cache') +} + +afterEach(async () => { + vi.useRealTimers() + await Promise.all( + temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true, force: true })) + ) +}) + +describe('SSH relay artifact cache eviction', () => { + it('pins the 2 GiB cap and bounded stable-tree scan', () => { + expect(SSH_RELAY_ARTIFACT_CACHE_EVICTION_LIMITS).toEqual({ + maximumBytes: 2 * 1024 * 1024 * 1024, + transactionTimeoutMs: 5 * 60_000, + maximumMembersPerEntry: 10_000 + }) + expect(Object.isFrozen(SSH_RELAY_ARTIFACT_CACHE_EVICTION_LIMITS)).toBe(true) + }) + + it('rejects invalid budgets and protected identities before touching cache state', async () => { + const root = await cacheRoot() + await expect(evictSshRelayArtifactCache({ cacheRoot: root, maximumBytes: -1 })).rejects.toThrow( + /maximum bytes/i + ) + await expect( + evictSshRelayArtifactCache({ + cacheRoot: root, + protectedContentIds: ['not-a-digest' as SshRelayDigest] + }) + ).rejects.toThrow(/content id/i) + }) + + it('accounts exact logical bytes and evicts least-recently-used idle content', async () => { + vi.useFakeTimers() + const root = await cacheRoot() + vi.setSystemTime(1_000) + const oldest = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'a' + }) + const firstLease = await acquireSshRelayArtifactCacheInUseLease({ + cacheRoot: root, + entry: oldest.entry + }) + await firstLease.release() + vi.setSystemTime(2_000) + const newest = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'b' + }) + const secondLease = await acquireSshRelayArtifactCacheInUseLease({ + cacheRoot: root, + entry: newest.entry + }) + await secondLease.release() + + const result = await evictSshRelayArtifactCache({ + cacheRoot: root, + maximumBytes: newest.logicalBytes + }) + + expect(result).toMatchObject({ + initialBytes: oldest.logicalBytes + newest.logicalBytes, + finalBytes: newest.logicalBytes, + reclaimedBytes: oldest.logicalBytes, + evictedContentIds: [oldest.entry.contentId] + }) + await expect(stat(oldest.entry.entryPath)).rejects.toMatchObject({ code: 'ENOENT' }) + await expect(stat(newest.entry.entryPath)).resolves.toMatchObject({ + isDirectory: expect.any(Function) + }) + }) + + it('never evicts active references and becomes evictable after every release', async () => { + const root = await cacheRoot() + const value = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'a' + }) + const first = await acquireSshRelayArtifactCacheInUseLease({ + cacheRoot: root, + entry: value.entry + }) + const second = await acquireSshRelayArtifactCacheInUseLease({ + cacheRoot: root, + entry: value.entry + }) + const retained = await evictSshRelayArtifactCache({ cacheRoot: root, maximumBytes: 0 }) + expect(retained).toMatchObject({ finalBytes: value.logicalBytes, evictedContentIds: [] }) + + await Promise.all([first.release(), second.release()]) + const removed = await evictSshRelayArtifactCache({ cacheRoot: root, maximumBytes: 0 }) + expect(removed.evictedContentIds).toEqual([value.entry.contentId]) + }) + + it('retains hard-protected content and prefers current/previous content while it fits', async () => { + const root = await cacheRoot() + const hard = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'a' + }) + const preferred = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'b' + }) + const ordinary = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'c' + }) + const result = await evictSshRelayArtifactCache({ + cacheRoot: root, + maximumBytes: hard.logicalBytes + preferred.logicalBytes, + protectedContentIds: [hard.entry.contentId], + preferredRetentionContentIds: [preferred.entry.contentId] + }) + expect(result.evictedContentIds).toEqual([ordinary.entry.contentId]) + + const forced = await evictSshRelayArtifactCache({ + cacheRoot: root, + maximumBytes: hard.logicalBytes, + protectedContentIds: [hard.entry.contentId], + preferredRetentionContentIds: [preferred.entry.contentId] + }) + expect(forced.evictedContentIds).toEqual([preferred.entry.contentId]) + expect(forced.finalBytes).toBe(hard.logicalBytes) + }) + + it('preserves malformed, live, and remote-host lease state as ambiguous', async () => { + const root = await cacheRoot() + const values = await Promise.all( + ['a', 'b', 'c'].map((character) => + createSshRelayArtifactCacheEvictionFixture({ cacheRoot: root, character }) + ) + ) + const owners = [ + '{not-json', + JSON.stringify({ + schemaVersion: 1, + contentId: values[1].entry.contentId, + token: 'd'.repeat(32), + hostname: hostname(), + pid: process.pid, + acquiredAtMs: 0, + heartbeatAtMs: 0 + }), + JSON.stringify({ + schemaVersion: 1, + contentId: values[2].entry.contentId, + token: 'e'.repeat(32), + hostname: 'another-host', + pid: 2_147_483_647, + acquiredAtMs: 0, + heartbeatAtMs: 0 + }) + ] + for (const [index, value] of values.entries()) { + const token = index === 0 ? 'f'.repeat(32) : index === 1 ? 'd'.repeat(32) : 'e'.repeat(32) + const leasePath = join(root, 'in-use', value.entry.contentId.slice(7), `${token}.lease`) + await mkdir(leasePath, { recursive: true }) + await writeFile(join(leasePath, 'owner.json'), `${owners[index]}\n`) + } + const result = await evictSshRelayArtifactCache({ cacheRoot: root, maximumBytes: 0 }) + expect(result.evictedContentIds).toEqual([]) + expect(result.finalBytes).toBe(values.reduce((total, value) => total + value.logicalBytes, 0)) + }) + + it('reclaims only a confirmed stale same-host dead lease before eviction', async () => { + const root = await cacheRoot() + const value = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'a' + }) + const token = 'd'.repeat(32) + const leasePath = join(root, 'in-use', value.entry.contentId.slice(7), `${token}.lease`) + await mkdir(leasePath, { recursive: true }) + await writeFile( + join(leasePath, 'owner.json'), + `${JSON.stringify({ + schemaVersion: 1, + contentId: value.entry.contentId, + token, + hostname: hostname(), + pid: 2_147_483_647, + acquiredAtMs: 0, + heartbeatAtMs: 0 + })}\n` + ) + const result = await evictSshRelayArtifactCache({ cacheRoot: root, maximumBytes: 0 }) + expect(result.evictedContentIds).toEqual([value.entry.contentId]) + await expect(stat(leasePath)).rejects.toMatchObject({ code: 'ENOENT' }) + }) + + it('preserves staging and safely settles cancellation behind a content lock', async () => { + const root = await cacheRoot() + const value = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'a' + }) + await mkdir(`${value.entry.entryPath}.pending-${'f'.repeat(32)}`) + const staged = await evictSshRelayArtifactCache({ cacheRoot: root, maximumBytes: 0 }) + expect(staged.evictedContentIds).toEqual([]) + + await rm(`${value.entry.entryPath}.pending-${'f'.repeat(32)}`, { recursive: true }) + const lock = await acquireSshRelayArtifactCacheLock({ + cacheRoot: root, + contentId: value.entry.contentId + }) + const controller = new AbortController() + const pending = evictSshRelayArtifactCache({ + cacheRoot: root, + maximumBytes: 0, + signal: controller.signal + }) + controller.abort(new Error('cancel eviction wait')) + await expect(pending).rejects.toThrow(/cancel eviction wait/i) + await lock.release() + await expect(stat(value.entry.entryPath)).resolves.toMatchObject({ + isDirectory: expect.any(Function) + }) + }) + + it('serializes concurrent evictors and protects an unsafe linked tree', async () => { + const root = await cacheRoot() + const first = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'a' + }) + const second = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'b' + }) + const [left, right] = await Promise.all([ + evictSshRelayArtifactCache({ cacheRoot: root, maximumBytes: second.logicalBytes }), + evictSshRelayArtifactCache({ cacheRoot: root, maximumBytes: second.logicalBytes }) + ]) + expect(new Set([...left.evictedContentIds, ...right.evictedContentIds])).toEqual( + new Set([first.entry.contentId]) + ) + + const unsafe = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'c' + }) + await symlink( + join(unsafe.entry.runtimeRoot, 'relay.js'), + join(unsafe.entry.runtimeRoot, 'link') + ) + const result = await evictSshRelayArtifactCache({ cacheRoot: root, maximumBytes: 0 }) + expect(result.evictedContentIds).not.toContain(unsafe.entry.contentId) + expect(await readFile(join(unsafe.entry.runtimeRoot, 'relay.js'))).toHaveLength(33) + }) +}) diff --git a/src/main/ssh/ssh-relay-artifact-cache-eviction.ts b/src/main/ssh/ssh-relay-artifact-cache-eviction.ts new file mode 100644 index 00000000000..6d8f703f32b --- /dev/null +++ b/src/main/ssh/ssh-relay-artifact-cache-eviction.ts @@ -0,0 +1,293 @@ +import { randomBytes } from 'node:crypto' +import { lstat, mkdir, readdir, rename, rm } from 'node:fs/promises' +import { join, resolve } from 'node:path' + +import { + measureSshRelayArtifactCacheEntryLogicalBytes, + SSH_RELAY_ARTIFACT_CACHE_MAXIMUM_MEMBERS_PER_ENTRY +} from './ssh-relay-artifact-cache-entry-accounting' +import { inspectSshRelayArtifactCacheInUse } from './ssh-relay-artifact-cache-in-use-inspection' +import { acquireSshRelayArtifactCacheLock } from './ssh-relay-artifact-cache-lock' +import { sshRelayArtifactCacheErrorCode } from './ssh-relay-artifact-cache-lock-record' +import { + readSshRelayArtifactCacheRecency, + removeSshRelayArtifactCacheRecency +} from './ssh-relay-artifact-cache-recency' +import type { SshRelayDigest } from './ssh-relay-runtime-identity' + +const MAXIMUM_BYTES = 2 * 1024 * 1024 * 1024 +const TRANSACTION_TIMEOUT_MS = 5 * 60_000 +const CONTENT_HEX = /^[0-9a-f]{64}$/ +const CONTENT_ID = /^sha256:[0-9a-f]{64}$/ +const EVICTION_LOCK_ID = `sha256:${'0'.repeat(64)}` as SshRelayDigest + +type CacheCandidate = { + contentId: SshRelayDigest + contentHex: string + entryPath: string + logicalBytes: number + usedAtMs: number + state: 'idle' | 'active' | 'ambiguous' | 'staging' +} + +function exactMaximumBytes(value: number): number { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error('SSH relay artifact cache maximum bytes must be a non-negative safe integer') + } + return value +} + +function exactContentSet(values: readonly SshRelayDigest[] | undefined): Set { + const result = new Set() + for (const value of values ?? []) { + if (!CONTENT_ID.test(value)) { + throw new Error('SSH relay artifact cache protected content ID must be an exact digest') + } + result.add(value) + } + return result +} + +async function hasSameContentStaging(entries: string, contentHex: string): Promise { + const pattern = new RegExp(`^${contentHex}\\.pending-[0-9a-f]{32}$`) + return (await readdir(entries)).some((name) => pattern.test(name)) +} + +async function inspectCandidate({ + cacheRoot, + entries, + contentHex, + signal +}: { + cacheRoot: string + entries: string + contentHex: string + signal: AbortSignal +}): Promise { + const contentId = `sha256:${contentHex}` as SshRelayDigest + const lock = await acquireSshRelayArtifactCacheLock({ cacheRoot, contentId, signal }) + try { + signal.throwIfAborted() + const entryPath = join(entries, contentHex) + const root = await lstat(entryPath).catch((error) => { + if (sshRelayArtifactCacheErrorCode(error) === 'ENOENT') { + return null + } + throw error + }) + if (!root) { + return null + } + const logicalBytes = await measureSshRelayArtifactCacheEntryLogicalBytes(entryPath, signal) + if (!root.isDirectory() || root.isSymbolicLink() || logicalBytes === null) { + return { + contentId, + contentHex, + entryPath, + logicalBytes: 0, + usedAtMs: Math.trunc(root.mtimeMs), + state: 'ambiguous' + } + } + const recency = await readSshRelayArtifactCacheRecency({ + cacheRoot, + contentId, + initialUsedAtMs: root.mtimeMs + }) + const inUse = await inspectSshRelayArtifactCacheInUse({ cacheRoot, contentId, signal }) + const staging = await hasSameContentStaging(entries, contentHex) + return { + contentId, + contentHex, + entryPath, + logicalBytes, + usedAtMs: recency.kind === 'known' ? recency.usedAtMs : Math.trunc(root.mtimeMs), + state: staging ? 'staging' : recency.kind === 'ambiguous' ? 'ambiguous' : inUse + } + } finally { + await lock.release() + } +} + +function candidateOrder( + preferred: ReadonlySet +): (left: CacheCandidate, right: CacheCandidate) => number { + return (left, right) => { + const preference = + Number(preferred.has(left.contentId)) - Number(preferred.has(right.contentId)) + return ( + preference || + left.usedAtMs - right.usedAtMs || + left.contentHex.localeCompare(right.contentHex) + ) + } +} + +async function deleteIdleCandidate({ + cacheRoot, + entries, + candidate, + signal +}: { + cacheRoot: string + entries: string + candidate: CacheCandidate + signal: AbortSignal +}): Promise { + const lock = await acquireSshRelayArtifactCacheLock({ + cacheRoot, + contentId: candidate.contentId, + signal + }) + try { + signal.throwIfAborted() + if (await hasSameContentStaging(entries, candidate.contentHex)) { + return null + } + const root = await lstat(candidate.entryPath).catch(() => null) + if (!root?.isDirectory() || root.isSymbolicLink()) { + return null + } + const recency = await readSshRelayArtifactCacheRecency({ + cacheRoot, + contentId: candidate.contentId, + initialUsedAtMs: root.mtimeMs + }) + // Why: a use that races the initial LRU scan wins; a later transaction can reconsider it. + if (recency.kind !== 'known' || recency.usedAtMs !== candidate.usedAtMs) { + return null + } + const inUse = await inspectSshRelayArtifactCacheInUse({ + cacheRoot, + contentId: candidate.contentId, + signal + }) + if (inUse !== 'idle') { + return null + } + const bytes = await measureSshRelayArtifactCacheEntryLogicalBytes(candidate.entryPath, signal) + if (bytes === null) { + return null + } + const tombstone = `${candidate.entryPath}.evicting-${randomBytes(16).toString('hex')}` + // Why: only an exact final entry is renamed unselectable before owned recursive deletion. + await rename(candidate.entryPath, tombstone) + await rm(tombstone, { recursive: true, force: true }) + await Promise.all([ + removeSshRelayArtifactCacheRecency(cacheRoot, candidate.contentId).catch(() => {}), + rm(resolve(cacheRoot, 'in-use', candidate.contentHex), { + recursive: true, + force: true + }).catch(() => {}) + ]) + return bytes + } catch (error) { + if (sshRelayArtifactCacheErrorCode(error) === 'ENOENT') { + return null + } + throw error + } finally { + await lock.release() + } +} + +export type SshRelayArtifactCacheEvictionResult = { + initialBytes: number + finalBytes: number + reclaimedBytes: number + evictedContentIds: SshRelayDigest[] + blockedContentIds: SshRelayDigest[] + accountingComplete: boolean +} + +export async function evictSshRelayArtifactCache({ + cacheRoot, + maximumBytes = MAXIMUM_BYTES, + protectedContentIds, + preferredRetentionContentIds, + signal +}: { + cacheRoot: string + maximumBytes?: number + protectedContentIds?: readonly SshRelayDigest[] + preferredRetentionContentIds?: readonly SshRelayDigest[] + signal?: AbortSignal +}): Promise { + const limit = exactMaximumBytes(maximumBytes) + const protectedSet = exactContentSet(protectedContentIds) + const preferredSet = exactContentSet(preferredRetentionContentIds) + const timeout = AbortSignal.timeout(TRANSACTION_TIMEOUT_MS) + const activeSignal = signal ? AbortSignal.any([signal, timeout]) : timeout + activeSignal.throwIfAborted() + const root = resolve(cacheRoot) + const entries = join(root, 'entries') + await mkdir(entries, { recursive: true, mode: 0o700 }) + // Why: the dedicated namespace serializes LRU transactions without reserving a content digest. + const transaction = await acquireSshRelayArtifactCacheLock({ + cacheRoot: join(root, 'eviction-transaction'), + contentId: EVICTION_LOCK_ID, + signal: activeSignal + }) + try { + const candidates: CacheCandidate[] = [] + for (const name of (await readdir(entries)).sort()) { + activeSignal.throwIfAborted() + if (!CONTENT_HEX.test(name)) { + continue + } + const candidate = await inspectCandidate({ + cacheRoot: root, + entries, + contentHex: name, + signal: activeSignal + }) + if (candidate) { + candidates.push(candidate) + } + } + const accountingComplete = candidates.every((candidate) => candidate.logicalBytes > 0) + const initialBytes = candidates.reduce((total, candidate) => total + candidate.logicalBytes, 0) + let finalBytes = initialBytes + const blocked = candidates + .filter((candidate) => candidate.state !== 'idle' || protectedSet.has(candidate.contentId)) + .map((candidate) => candidate.contentId) + const evictable = candidates + .filter((candidate) => candidate.state === 'idle' && !protectedSet.has(candidate.contentId)) + .sort(candidateOrder(preferredSet)) + const evictedContentIds: SshRelayDigest[] = [] + for (const candidate of evictable) { + if (finalBytes <= limit) { + break + } + activeSignal.throwIfAborted() + const reclaimed = await deleteIdleCandidate({ + cacheRoot: root, + entries, + candidate, + signal: activeSignal + }) + if (reclaimed === null) { + blocked.push(candidate.contentId) + continue + } + finalBytes -= reclaimed + evictedContentIds.push(candidate.contentId) + } + return { + initialBytes, + finalBytes, + reclaimedBytes: initialBytes - finalBytes, + evictedContentIds, + blockedContentIds: [...new Set(blocked)], + accountingComplete + } + } finally { + await transaction.release() + } +} + +export const SSH_RELAY_ARTIFACT_CACHE_EVICTION_LIMITS = Object.freeze({ + maximumBytes: MAXIMUM_BYTES, + transactionTimeoutMs: TRANSACTION_TIMEOUT_MS, + maximumMembersPerEntry: SSH_RELAY_ARTIFACT_CACHE_MAXIMUM_MEMBERS_PER_ENTRY +}) diff --git a/src/main/ssh/ssh-relay-artifact-cache-in-use-inspection.ts b/src/main/ssh/ssh-relay-artifact-cache-in-use-inspection.ts new file mode 100644 index 00000000000..7e969cf9b55 --- /dev/null +++ b/src/main/ssh/ssh-relay-artifact-cache-in-use-inspection.ts @@ -0,0 +1,87 @@ +import { randomBytes } from 'node:crypto' +import { lstat, readdir, rename, rm } from 'node:fs/promises' +import { hostname } from 'node:os' +import { join, resolve } from 'node:path' + +import { + readSshRelayArtifactCacheInUseOwner, + sameSshRelayArtifactCacheInUseDirectory +} from './ssh-relay-artifact-cache-in-use-record' +import { sshRelayArtifactCacheErrorCode } from './ssh-relay-artifact-cache-lock-record' +import type { SshRelayDigest } from './ssh-relay-runtime-identity' + +export const SSH_RELAY_ARTIFACT_CACHE_IN_USE_STALE_AFTER_MS = 30_000 +const CONTENT_ID = /^sha256:([0-9a-f]{64})$/ +const LEASE_NAME = /^([0-9a-f]{32})\.lease$/ + +function exactContentHex(contentId: SshRelayDigest): string { + const match = CONTENT_ID.exec(contentId) + if (!match) { + throw new Error('SSH relay artifact cache in-use content ID must be an exact lowercase digest') + } + return match[1] +} + +function localProcessIsAlive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch (error) { + return sshRelayArtifactCacheErrorCode(error) !== 'ESRCH' + } +} + +export async function inspectSshRelayArtifactCacheInUse({ + cacheRoot, + contentId, + signal +}: { + cacheRoot: string + contentId: SshRelayDigest + signal: AbortSignal +}): Promise<'idle' | 'active' | 'ambiguous'> { + const parent = resolve(cacheRoot, 'in-use', exactContentHex(contentId)) + let names: string[] + try { + names = await readdir(parent) + } catch (error) { + return sshRelayArtifactCacheErrorCode(error) === 'ENOENT' ? 'idle' : 'ambiguous' + } + for (const name of names) { + signal.throwIfAborted() + const match = LEASE_NAME.exec(name) + if (!match) { + return 'ambiguous' + } + const token = match[1] + const leasePath = join(parent, name) + const directory = await lstat(leasePath, { bigint: true }).catch(() => null) + const owner = await readSshRelayArtifactCacheInUseOwner(leasePath, contentId, token) + if (!directory?.isDirectory() || directory.isSymbolicLink() || !owner) { + return 'ambiguous' + } + if (Date.now() - owner.heartbeatAtMs < SSH_RELAY_ARTIFACT_CACHE_IN_USE_STALE_AFTER_MS) { + return 'active' + } + if (owner.hostname !== hostname()) { + return 'ambiguous' + } + if (localProcessIsAlive(owner.pid)) { + return 'active' + } + const confirmed = await readSshRelayArtifactCacheInUseOwner(leasePath, contentId, token) + const current = await lstat(leasePath, { bigint: true }).catch(() => null) + if ( + !confirmed || + confirmed.heartbeatAtMs !== owner.heartbeatAtMs || + !current || + !sameSshRelayArtifactCacheInUseDirectory(directory, current) + ) { + return 'ambiguous' + } + const tombstone = `${leasePath}.stale-${randomBytes(16).toString('hex')}` + await rename(leasePath, tombstone) + await rm(tombstone, { recursive: true, force: true }) + } + return 'idle' +} diff --git a/src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts b/src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts new file mode 100644 index 00000000000..a92ed8cb738 --- /dev/null +++ b/src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts @@ -0,0 +1,178 @@ +import { spawn } from 'node:child_process' +import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' + +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { createSshRelayArtifactCacheEvictionFixture } from './ssh-relay-artifact-cache-eviction-fixture' +import { evictSshRelayArtifactCache } from './ssh-relay-artifact-cache-eviction' +import { + acquireSshRelayArtifactCacheInUseLease, + sshRelayArtifactCacheInUseLeasePath, + SSH_RELAY_ARTIFACT_CACHE_IN_USE_LIMITS +} from './ssh-relay-artifact-cache-in-use-lease' +import type { SshRelayDigest } from './ssh-relay-runtime-identity' + +const temporaryDirectories: string[] = [] +const leaseModuleUrl = pathToFileURL( + join(import.meta.dirname, 'ssh-relay-artifact-cache-in-use-lease.ts') +).href + +async function cacheRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-relay-cache-in-use-')) + temporaryDirectories.push(root) + return join(root, 'cache') +} + +afterEach(async () => { + vi.useRealTimers() + await Promise.all( + temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true, force: true })) + ) +}) + +describe('SSH relay artifact cache in-use lease', () => { + it('pins exact paths and bounded heartbeat/stale timing', async () => { + const root = await cacheRoot() + const { entry } = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'a' + }) + const token = 'b'.repeat(32) + expect(sshRelayArtifactCacheInUseLeasePath(root, entry.contentId, token)).toBe( + join(root, 'in-use', 'a'.repeat(64), `${token}.lease`) + ) + expect(SSH_RELAY_ARTIFACT_CACHE_IN_USE_LIMITS).toEqual({ + heartbeatIntervalMs: 5_000, + staleAfterMs: 30_000, + acquisitionTimeoutMs: 120_000 + }) + expect(Object.isFrozen(SSH_RELAY_ARTIFACT_CACHE_IN_USE_LIMITS)).toBe(true) + for (const invalid of [ + `sha256:${'A'.repeat(64)}`, + `sha256:${'a'.repeat(63)}`, + `${entry.contentId}/../escape` + ]) { + expect(() => + sshRelayArtifactCacheInUseLeasePath(root, invalid as SshRelayDigest, token) + ).toThrow(/content id/i) + } + expect(() => sshRelayArtifactCacheInUseLeasePath(root, entry.contentId, '../escape')).toThrow( + /token/i + ) + }) + + it('allows multiple references and heartbeats owner records independently', async () => { + vi.useFakeTimers() + vi.setSystemTime(100_000) + const root = await cacheRoot() + const { entry } = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'a' + }) + const first = await acquireSshRelayArtifactCacheInUseLease({ cacheRoot: root, entry }) + const second = await acquireSshRelayArtifactCacheInUseLease({ cacheRoot: root, entry }) + expect(second.token).not.toBe(first.token) + + await vi.advanceTimersByTimeAsync(5_000) + await first.assertOwned() + const owner = JSON.parse(await readFile(join(first.leasePath, 'owner.json'), 'utf8')) as { + heartbeatAtMs: number + } + expect(owner.heartbeatAtMs).toBe(105_000) + await Promise.all([first.release(), second.release()]) + expect(await readdir(join(root, 'recency', entry.contentId.slice(7)))).toHaveLength(1) + }) + + it('does not delete a displaced successor through owner release', async () => { + const root = await cacheRoot() + const { entry } = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'a' + }) + const lease = await acquireSshRelayArtifactCacheInUseLease({ cacheRoot: root, entry }) + const successor = { token: 'c'.repeat(32) } + const owner = JSON.parse(await readFile(join(lease.leasePath, 'owner.json'), 'utf8')) as object + await writeFile( + join(lease.leasePath, 'owner.json'), + `${JSON.stringify({ ...owner, ...successor })}\n` + ) + + await expect(lease.assertOwned()).rejects.toThrow(/ownership/i) + await lease.release() + await expect(stat(lease.leasePath)).resolves.toMatchObject({ + isDirectory: expect.any(Function) + }) + }) + + it('makes a live lease visible to a real second process', async () => { + const root = await cacheRoot() + const { entry } = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'a' + }) + const script = ` + import { registerHooks } from 'node:module' + registerHooks({ resolve(specifier, context, nextResolve) { + try { return nextResolve(specifier, context) } + catch (error) { if (specifier.startsWith('.')) return nextResolve(specifier + '.ts', context); throw error } + } }) + const { acquireSshRelayArtifactCacheInUseLease } = await import(${JSON.stringify(leaseModuleUrl)}) + const entry = JSON.parse(process.env.ORCA_RELAY_CACHE_ENTRY) + const lease = await acquireSshRelayArtifactCacheInUseLease({ cacheRoot: process.env.ORCA_RELAY_CACHE_ROOT, entry }) + process.stdout.write(lease.leasePath + '\\n') + await new Promise(resolve => process.stdin.once('data', resolve)) + await lease.release() + ` + const child = spawn( + process.execPath, + [ + '--experimental-strip-types', + '--disable-warning=MODULE_TYPELESS_PACKAGE_JSON', + '--input-type=module', + '--eval', + script + ], + { + cwd: process.cwd(), + env: { + ...process.env, + ORCA_RELAY_CACHE_ROOT: root, + ORCA_RELAY_CACHE_ENTRY: JSON.stringify(entry) + }, + stdio: ['pipe', 'pipe', 'pipe'] + } + ) + let childError = '' + child.stderr.on('data', (bytes: Buffer) => (childError += bytes.toString('utf8'))) + const leasePath = await new Promise((resolve, reject) => { + child.once('error', reject) + child.stdout.once('data', (bytes: Buffer) => resolve(bytes.toString('utf8').trim())) + child.once('exit', (code) => reject(new Error(`Lease child exited ${code}: ${childError}`))) + }) + await expect(stat(leasePath)).resolves.toMatchObject({ isDirectory: expect.any(Function) }) + const retained = await evictSshRelayArtifactCache({ cacheRoot: root, maximumBytes: 0 }) + expect(retained.evictedContentIds).toEqual([]) + child.stdin.end('release\n') + const code = await new Promise((resolve) => child.once('exit', resolve)) + expect(code).toBe(0) + }) + + it('rejects a reference when the exact final entry no longer exists', async () => { + const root = await cacheRoot() + const { entry } = await createSshRelayArtifactCacheEvictionFixture({ + cacheRoot: root, + character: 'a' + }) + await rm(entry.entryPath, { recursive: true }) + await expect( + acquireSshRelayArtifactCacheInUseLease({ cacheRoot: root, entry }) + ).rejects.toThrow(/entry/i) + await mkdir(join(root, 'in-use'), { recursive: true }) + expect(sshRelayArtifactCacheInUseLeasePath(root, entry.contentId, 'd'.repeat(32))).toContain( + join('in-use', 'a'.repeat(64)) + ) + }) +}) diff --git a/src/main/ssh/ssh-relay-artifact-cache-in-use-lease.ts b/src/main/ssh/ssh-relay-artifact-cache-in-use-lease.ts new file mode 100644 index 00000000000..4a1603d01c5 --- /dev/null +++ b/src/main/ssh/ssh-relay-artifact-cache-in-use-lease.ts @@ -0,0 +1,258 @@ +import { randomBytes } from 'node:crypto' +import { lstat, mkdir, open, rename, rm, type FileHandle } from 'node:fs/promises' +import { hostname } from 'node:os' +import { dirname, join, resolve } from 'node:path' + +import type { SshRelayArtifactCacheEntry } from './ssh-relay-artifact-cache-entry-verification' +import { SSH_RELAY_ARTIFACT_CACHE_IN_USE_STALE_AFTER_MS } from './ssh-relay-artifact-cache-in-use-inspection' +import { + readSshRelayArtifactCacheInUseOwner, + sameSshRelayArtifactCacheInUseDirectory, + sshRelayArtifactCacheInUseOwnerBytes, + SSH_RELAY_ARTIFACT_CACHE_IN_USE_SCHEMA_VERSION, + type SshRelayArtifactCacheInUseDirectoryIdentity, + type SshRelayArtifactCacheInUseOwner +} from './ssh-relay-artifact-cache-in-use-record' +import { acquireSshRelayArtifactCacheLock } from './ssh-relay-artifact-cache-lock' +import { sshRelayArtifactCacheErrorCode } from './ssh-relay-artifact-cache-lock-record' +import { recordSshRelayArtifactCacheRecency } from './ssh-relay-artifact-cache-recency' +import type { SshRelayDigest } from './ssh-relay-runtime-identity' + +const HEARTBEAT_INTERVAL_MS = 5_000 +const ACQUISITION_TIMEOUT_MS = 120_000 +const CONTENT_ID = /^sha256:([0-9a-f]{64})$/ +const TOKEN = /^[0-9a-f]{32}$/ + +function exactContentHex(contentId: SshRelayDigest): string { + const match = CONTENT_ID.exec(contentId) + if (!match) { + throw new Error('SSH relay artifact cache in-use content ID must be an exact lowercase digest') + } + return match[1] +} + +export function sshRelayArtifactCacheInUseLeasePath( + cacheRoot: string, + contentId: SshRelayDigest, + token: string +): string { + if (!TOKEN.test(token)) { + throw new Error('SSH relay artifact cache in-use token must be exact lowercase hex') + } + return resolve(cacheRoot, 'in-use', exactContentHex(contentId), `${token}.lease`) +} + +async function writeCompleteOwner(handle: FileHandle, owner: SshRelayArtifactCacheInUseOwner) { + const bytes = sshRelayArtifactCacheInUseOwnerBytes(owner) + let offset = 0 + while (offset < bytes.length) { + const { bytesWritten } = await handle.write(bytes, offset, bytes.length - offset, offset) + if (bytesWritten <= 0) { + throw new Error('SSH relay artifact cache in-use heartbeat could not be persisted') + } + offset += bytesWritten + } + await handle.truncate(bytes.length) + await handle.sync() +} + +class SshRelayArtifactCacheInUseLeaseOwner { + readonly leasePath: string + readonly token: string + private readonly cacheRoot: string + private readonly contentId: SshRelayDigest + private readonly directory: SshRelayArtifactCacheInUseDirectoryIdentity + private readonly ownerHandle: FileHandle + private readonly owner: SshRelayArtifactCacheInUseOwner + private heartbeat: Promise = Promise.resolve() + private heartbeatError: unknown + private heartbeatTimer: NodeJS.Timeout | undefined + private released = false + + constructor(options: { + cacheRoot: string + leasePath: string + contentId: SshRelayDigest + token: string + directory: SshRelayArtifactCacheInUseDirectoryIdentity + ownerHandle: FileHandle + owner: SshRelayArtifactCacheInUseOwner + }) { + this.cacheRoot = options.cacheRoot + this.leasePath = options.leasePath + this.contentId = options.contentId + this.token = options.token + this.directory = options.directory + this.ownerHandle = options.ownerHandle + this.owner = options.owner + this.heartbeatTimer = setInterval(() => this.queueHeartbeat(), HEARTBEAT_INTERVAL_MS) + this.heartbeatTimer.unref() + } + + private queueHeartbeat(): void { + this.heartbeat = this.heartbeat.then(async () => { + if (this.released || this.heartbeatError) { + return + } + try { + await this.assertPathOwnership() + this.owner.heartbeatAtMs = Date.now() + await writeCompleteOwner(this.ownerHandle, this.owner) + } catch (error) { + this.heartbeatError = error + } + }) + } + + private async assertPathOwnership(): Promise { + const metadata = await lstat(this.leasePath, { bigint: true }).catch(() => null) + const current = await readSshRelayArtifactCacheInUseOwner( + this.leasePath, + this.contentId, + this.token + ) + if ( + !metadata?.isDirectory() || + !sameSshRelayArtifactCacheInUseDirectory(this.directory, metadata) || + current?.token !== this.token + ) { + throw new Error('SSH relay artifact cache in-use ownership was displaced') + } + } + + async assertOwned(): Promise { + await this.heartbeat + if (this.released || this.heartbeatError) { + throw new Error('SSH relay artifact cache in-use ownership is no longer active', { + cause: this.heartbeatError + }) + } + await this.assertPathOwnership() + } + + async release(): Promise { + if (this.released) { + return + } + this.released = true + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer) + } + this.heartbeatTimer = undefined + await this.heartbeat + let owned = false + try { + await this.assertPathOwnership() + owned = true + } catch { + // A displaced lease must never remove another process's visible reference. + } + await this.ownerHandle.close() + if (!owned) { + return + } + const tombstone = `${this.leasePath}.released-${this.token}` + await rename(this.leasePath, tombstone) + await rm(tombstone, { recursive: true, force: true }) + await recordSshRelayArtifactCacheRecency({ + cacheRoot: this.cacheRoot, + contentId: this.contentId + }) + } +} + +export type SshRelayArtifactCacheInUseLease = Pick< + SshRelayArtifactCacheInUseLeaseOwner, + 'leasePath' | 'token' | 'assertOwned' | 'release' +> + +async function createLease( + cacheRoot: string, + contentId: SshRelayDigest +): Promise { + const token = randomBytes(16).toString('hex') + const leasePath = sshRelayArtifactCacheInUseLeasePath(cacheRoot, contentId, token) + await mkdir(dirname(leasePath), { recursive: true, mode: 0o700 }) + await mkdir(leasePath, { mode: 0o700 }) + let handle: FileHandle | undefined + try { + handle = await open(join(leasePath, 'owner.json'), 'wx+', 0o600) + const now = Date.now() + const owner: SshRelayArtifactCacheInUseOwner = { + schemaVersion: SSH_RELAY_ARTIFACT_CACHE_IN_USE_SCHEMA_VERSION, + contentId, + token, + hostname: hostname(), + pid: process.pid, + acquiredAtMs: now, + heartbeatAtMs: now + } + await writeCompleteOwner(handle, owner) + const directory = await lstat(leasePath, { bigint: true }) + return new SshRelayArtifactCacheInUseLeaseOwner({ + cacheRoot, + leasePath, + contentId, + token, + directory, + ownerHandle: handle, + owner + }) + } catch (error) { + await handle?.close().catch(() => {}) + await rm(leasePath, { recursive: true, force: true }).catch(() => {}) + throw error + } +} + +export async function acquireSshRelayArtifactCacheInUseLease({ + cacheRoot, + entry, + signal +}: { + cacheRoot: string + entry: SshRelayArtifactCacheEntry + signal?: AbortSignal +}): Promise { + const timeout = AbortSignal.timeout(ACQUISITION_TIMEOUT_MS) + const activeSignal = signal ? AbortSignal.any([signal, timeout]) : timeout + activeSignal.throwIfAborted() + const expectedPath = resolve(cacheRoot, 'entries', exactContentHex(entry.contentId)) + if (resolve(entry.entryPath) !== expectedPath) { + throw new Error( + 'SSH relay artifact cache in-use entry path disagrees with its content identity' + ) + } + const lock = await acquireSshRelayArtifactCacheLock({ + cacheRoot, + contentId: entry.contentId, + signal: activeSignal + }) + try { + const metadata = await lstat(expectedPath).catch((error) => { + if (sshRelayArtifactCacheErrorCode(error) === 'ENOENT') { + throw new Error('SSH relay artifact cache in-use entry no longer exists', { cause: error }) + } + throw error + }) + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new Error('SSH relay artifact cache in-use entry is not a complete final directory') + } + const lease = await createLease(cacheRoot, entry.contentId) + try { + await recordSshRelayArtifactCacheRecency({ cacheRoot, contentId: entry.contentId }) + return lease + } catch (error) { + await lease.release().catch(() => {}) + throw error + } + } finally { + await lock.release() + } +} + +export const SSH_RELAY_ARTIFACT_CACHE_IN_USE_LIMITS = Object.freeze({ + heartbeatIntervalMs: HEARTBEAT_INTERVAL_MS, + staleAfterMs: SSH_RELAY_ARTIFACT_CACHE_IN_USE_STALE_AFTER_MS, + acquisitionTimeoutMs: ACQUISITION_TIMEOUT_MS +}) diff --git a/src/main/ssh/ssh-relay-artifact-cache-in-use-record.ts b/src/main/ssh/ssh-relay-artifact-cache-in-use-record.ts new file mode 100644 index 00000000000..c1a83d905fd --- /dev/null +++ b/src/main/ssh/ssh-relay-artifact-cache-in-use-record.ts @@ -0,0 +1,113 @@ +import { lstat, open } from 'node:fs/promises' +import { join } from 'node:path' + +import { sshRelayArtifactCacheErrorCode } from './ssh-relay-artifact-cache-lock-record' +import type { SshRelayDigest } from './ssh-relay-runtime-identity' + +export const SSH_RELAY_ARTIFACT_CACHE_IN_USE_SCHEMA_VERSION = 1 +const MAXIMUM_RECORD_BYTES = 16 * 1024 + +export type SshRelayArtifactCacheInUseOwner = { + schemaVersion: typeof SSH_RELAY_ARTIFACT_CACHE_IN_USE_SCHEMA_VERSION + contentId: SshRelayDigest + token: string + hostname: string + pid: number + acquiredAtMs: number + heartbeatAtMs: number +} + +export type SshRelayArtifactCacheInUseDirectoryIdentity = { dev: bigint; ino: bigint } + +function validOwner( + value: unknown, + contentId: SshRelayDigest, + token: string +): value is SshRelayArtifactCacheInUseOwner { + if (!value || typeof value !== 'object') { + return false + } + const owner = value as Partial + return ( + owner.schemaVersion === SSH_RELAY_ARTIFACT_CACHE_IN_USE_SCHEMA_VERSION && + owner.contentId === contentId && + owner.token === token && + /^[0-9a-f]{32}$/.test(owner.token) && + typeof owner.hostname === 'string' && + owner.hostname.length > 0 && + typeof owner.pid === 'number' && + Number.isSafeInteger(owner.pid) && + owner.pid > 0 && + typeof owner.acquiredAtMs === 'number' && + Number.isSafeInteger(owner.acquiredAtMs) && + owner.acquiredAtMs >= 0 && + typeof owner.heartbeatAtMs === 'number' && + Number.isSafeInteger(owner.heartbeatAtMs) && + owner.heartbeatAtMs >= owner.acquiredAtMs + ) +} + +export function sshRelayArtifactCacheInUseOwnerBytes( + owner: SshRelayArtifactCacheInUseOwner +): Buffer { + return Buffer.from(`${JSON.stringify(owner)}\n`, 'utf8') +} + +export async function readSshRelayArtifactCacheInUseOwner( + leasePath: string, + contentId: SshRelayDigest, + token: string +): Promise { + const path = join(leasePath, 'owner.json') + let metadata + try { + metadata = await lstat(path, { bigint: true }) + } catch (error) { + if (sshRelayArtifactCacheErrorCode(error) === 'ENOENT') { + return null + } + throw error + } + if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > MAXIMUM_RECORD_BYTES) { + return null + } + const handle = await open(path, 'r') + try { + const opened = await handle.stat({ bigint: true }) + if ( + !opened.isFile() || + opened.dev !== metadata.dev || + opened.ino !== metadata.ino || + opened.size !== metadata.size + ) { + return null + } + const bytes = Buffer.alloc(Number(opened.size)) + const { bytesRead } = await handle.read(bytes, 0, bytes.length, 0) + const after = await handle.stat({ bigint: true }) + if ( + bytesRead !== bytes.length || + after.size !== opened.size || + after.mtimeNs !== opened.mtimeNs || + after.ctimeNs !== opened.ctimeNs + ) { + return null + } + let input: unknown + try { + input = JSON.parse(bytes.toString('utf8')) + } catch { + return null + } + return validOwner(input, contentId, token) ? input : null + } finally { + await handle.close() + } +} + +export function sameSshRelayArtifactCacheInUseDirectory( + left: SshRelayArtifactCacheInUseDirectoryIdentity, + right: SshRelayArtifactCacheInUseDirectoryIdentity +): boolean { + return left.dev === right.dev && left.ino === right.ino +} diff --git a/src/main/ssh/ssh-relay-artifact-cache-recency.ts b/src/main/ssh/ssh-relay-artifact-cache-recency.ts new file mode 100644 index 00000000000..9d572f84f2d --- /dev/null +++ b/src/main/ssh/ssh-relay-artifact-cache-recency.ts @@ -0,0 +1,113 @@ +import { randomBytes } from 'node:crypto' +import { lstat, mkdir, open, readdir, rm } from 'node:fs/promises' +import { join, resolve } from 'node:path' + +import { sshRelayArtifactCacheErrorCode } from './ssh-relay-artifact-cache-lock-record' +import type { SshRelayDigest } from './ssh-relay-runtime-identity' + +const CONTENT_ID = /^sha256:([0-9a-f]{64})$/ +const RECORD_NAME = /^(\d{16})-([0-9a-f]{32})\.used$/ + +function exactContentHex(contentId: SshRelayDigest): string { + const match = CONTENT_ID.exec(contentId) + if (!match) { + throw new Error('SSH relay artifact cache recency content ID must be an exact lowercase digest') + } + return match[1] +} + +function exactUsedAtMs(value: number): number { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error('SSH relay artifact cache recency time must be a non-negative safe integer') + } + return value +} + +function recencyDirectory(cacheRoot: string, contentId: SshRelayDigest): string { + return resolve(cacheRoot, 'recency', exactContentHex(contentId)) +} + +export async function recordSshRelayArtifactCacheRecency({ + cacheRoot, + contentId, + usedAtMs = Date.now() +}: { + cacheRoot: string + contentId: SshRelayDigest + usedAtMs?: number +}): Promise { + const directory = recencyDirectory(cacheRoot, contentId) + await mkdir(directory, { recursive: true, mode: 0o700 }) + const timestamp = String(exactUsedAtMs(usedAtMs)).padStart(16, '0') + const path = join(directory, `${timestamp}-${randomBytes(16).toString('hex')}.used`) + const handle = await open(path, 'wx', 0o600) + try { + await handle.sync() + } finally { + await handle.close() + } + const names = await readdir(directory) + const valid = names.filter((name) => RECORD_NAME.test(name)).sort() + const newest = valid.at(-1) + await Promise.all( + valid + .filter((name) => name !== newest) + .map((name) => + rm(join(directory, name), { force: true }).catch((error) => { + if (sshRelayArtifactCacheErrorCode(error) !== 'ENOENT') { + throw error + } + }) + ) + ) +} + +export type SshRelayArtifactCacheRecency = + | { kind: 'known'; usedAtMs: number } + | { kind: 'ambiguous' } + +export async function readSshRelayArtifactCacheRecency({ + cacheRoot, + contentId, + initialUsedAtMs +}: { + cacheRoot: string + contentId: SshRelayDigest + initialUsedAtMs: number +}): Promise { + let names: string[] + try { + names = await readdir(recencyDirectory(cacheRoot, contentId)) + } catch (error) { + if (sshRelayArtifactCacheErrorCode(error) === 'ENOENT') { + return { kind: 'known', usedAtMs: exactUsedAtMs(Math.trunc(initialUsedAtMs)) } + } + return { kind: 'ambiguous' } + } + let usedAtMs = exactUsedAtMs(Math.trunc(initialUsedAtMs)) + for (const name of names) { + const match = RECORD_NAME.exec(name) + if (!match) { + return { kind: 'ambiguous' } + } + const metadata = await lstat(join(recencyDirectory(cacheRoot, contentId), name)).catch( + () => null + ) + if (!metadata?.isFile() || metadata.isSymbolicLink() || metadata.size !== 0) { + return { kind: 'ambiguous' } + } + const timestamp = Number(match[1]) + if (!Number.isSafeInteger(timestamp)) { + return { kind: 'ambiguous' } + } + usedAtMs = Math.max(usedAtMs, timestamp) + } + return { kind: 'known', usedAtMs } +} + +export async function removeSshRelayArtifactCacheRecency( + cacheRoot: string, + contentId: SshRelayDigest +): Promise { + await rm(recencyDirectory(cacheRoot, contentId), { recursive: true, force: true }) +}