diff --git a/.github/workflows/ssh-relay-runtime-artifacts.yml b/.github/workflows/ssh-relay-runtime-artifacts.yml index a9bf767ca86..42549c3eca4 100644 --- a/.github/workflows/ssh-relay-runtime-artifacts.yml +++ b/.github/workflows/ssh-relay-runtime-artifacts.yml @@ -264,6 +264,7 @@ jobs: src/main/ssh/ssh-relay-artifact-selector.test.ts \ src/main/ssh/ssh-relay-artifact-download.test.ts \ src/main/ssh/ssh-relay-artifact-extraction.test.ts \ + src/main/ssh/ssh-relay-artifact-cache-lock-release.test.ts \ src/main/ssh/ssh-relay-artifact-cache-lock.test.ts \ src/main/ssh/ssh-relay-artifact-cache-entry.test.ts \ src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts \ @@ -806,6 +807,7 @@ jobs: src/main/ssh/ssh-relay-artifact-selector.test.ts ` src/main/ssh/ssh-relay-artifact-download.test.ts ` src/main/ssh/ssh-relay-artifact-extraction.test.ts ` + src/main/ssh/ssh-relay-artifact-cache-lock-release.test.ts ` src/main/ssh/ssh-relay-artifact-cache-lock.test.ts ` src/main/ssh/ssh-relay-artifact-cache-entry.test.ts ` src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts ` diff --git a/config/scripts/ssh-relay-runtime-workflow.test.mjs b/config/scripts/ssh-relay-runtime-workflow.test.mjs index 7fe2346c326..b76eb938a0e 100644 --- a/config/scripts/ssh-relay-runtime-workflow.test.mjs +++ b/config/scripts/ssh-relay-runtime-workflow.test.mjs @@ -122,6 +122,7 @@ describe('SSH relay runtime artifact workflow', () => { 'ssh-relay-artifact-selector.test.ts', 'ssh-relay-artifact-download.test.ts', 'ssh-relay-artifact-extraction.test.ts', + 'ssh-relay-artifact-cache-lock-release.test.ts', 'ssh-relay-artifact-cache-lock.test.ts', 'ssh-relay-artifact-cache-entry.test.ts', 'ssh-relay-artifact-cache-in-use-lease.test.ts', diff --git a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md index 82637b67806..35517a07004 100644 --- a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md +++ b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist-summary.md @@ -9,9 +9,10 @@ keeps commands, hashes, runner identities, timings, and failure details. A checked box means the work has evidence in the detailed ledger. Design approval alone does not complete a box. -Active checkpoint: **Milestone 5 / Work Package 4 desktop resolver/cache — disconnected cold-cache -population locally green and wired; commit/all-six native proof next, 2026-07-15, Codex -implementation owner.** `E-M5-ARTIFACT-CACHE-RESOLUTION-CI-001` closes warm-cache commit +Active checkpoint: **Milestone 5 / Work Package 4 desktop resolver/cache — prove the locally green +Windows active-owner cache-lock release correction on replacement all-six native CI before +accepting cold-cache population, 2026-07-15, Codex implementation owner.** +`E-M5-ARTIFACT-CACHE-RESOLUTION-CI-001` closes warm-cache commit `22031fa68` on all six native clients, both Linux supplements, Windows x64 baseline, PR Checks, Golden E2E, and computer-use; Windows arm64 remains the expected build-26200 rejection against 26100. `E-M5-ARTIFACT-CACHE-RESOLUTION-LOCAL-001` passes 9/9 purpose tests, 16/16 focused+workflow-oracle tests, 323/323 non-full-size SSH relay tests, 280/280 release-script @@ -38,6 +39,24 @@ implementation now canonicalizes lease identity/locking/recency to the existing `E-M5-ARTIFACT-CACHE-PHYSICAL-ROOT-CORRECTION-LOCAL-001`: 7/7 correction tests and 21/21 focused+ workflow-oracle tests pass while missing/misplaced entries remain rejected. Broader exact-head proof is still required before checkpointing. +Exact-head cold-cache artifact run 29475848463 then repeated the prior Windows x64 active-owner +release race in job 87548519537: the new real Linux/Windows cold→warm integration cases pass, but +lock tombstone rename receives `EPERM` and the live waiter later receives teardown `ENOENT` +(`E-M5-ARTIFACT-CACHE-LOCK-RELEASE-CI-RED-001`). The prior ledger explicitly required correction if +this recurred. The bounded sharing-error correction below is locally green, but replacement all-six +native proof remains required; do not advance to acquisition, Electron/startup, SSH, settings, +fallback, tuple, publication, or default behavior. +`E-M5-ARTIFACT-CACHE-LOCK-RELEASE-LOCAL-RED-001` records the expected missing-module failure for the +fixed bounded retry/displacement/exhaustion contract before implementation. +`E-M5-ARTIFACT-CACHE-LOCK-RELEASE-LOCAL-001` is now locally green: only `EPERM`/`EACCES` rename +failures retry for at most 5 s at 50 ms intervals; every retry rechecks exact directory/nonce +ownership, displacement preserves the successor, absent paths settle, and unexpected or exhausted +failures propagate closed. Focused plus workflow-oracle proof passes 24/24, broader non-full-size +relay proof passes 339/339 with the three declared full-size skips, release scripts pass 280/280, +and typecheck, lint, format, diff, and protected-resolver gates pass. A replacement exact-head +all-six native run plus adjacent PR/E2E checks is still mandatory before this correction or the +cold-cache population package is accepted; do not advance to acquisition, Electron/startup, SSH, +settings, fallback, tuple, publication, or default behavior yet. `E-M5-ARTIFACT-CACHE-ROOT-CI-001` closes the pure cache-root contract at exact head `aefcaa9a9`: all six primary native Node 24 jobs, both Linux supplements, Windows x64 baseline, PR Checks, Golden E2E, and computer-use pass; Windows arm64 build 26200 remains correctly gated against 26100. The diff --git a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md index 8258032198f..8338be33452 100644 --- a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md +++ b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md @@ -8,8 +8,8 @@ work; keep exact commands, runner identities, hashes, metrics, and residual gaps Date created: 2026-07-14
Last updated: 2026-07-15
-Current phase: Milestone 5 / Work Package 4 desktop resolver/cache — **In progress — 2026-07-15, Codex implementation owner: prove disconnected cold-cache population**. Require the eventual startup caller to supply Orca's pre-`app.setName()` canonical user-data path; keep filesystem orchestration outside import time and production manifest/resources, proxy, SSH/settings/tuples/release publication/default behavior absent. Real Apple/SignPath rehearsals remain an explicit late gate and are not on this package's critical path.
-Session checkpoint: **In progress — 2026-07-15, Codex implementation owner** — `E-M5-ARTIFACT-CACHE-RESOLUTION-CI-001` closes warm-cache commit `22031fa686d2a7c96e687bc02c16fdb08bc987d7` on all six primary native clients, both Linux supplements, Windows x64 baseline, PR Checks, Golden E2E, and computer-use; Windows arm64 remains the expected build-26200 rejection against 26100. The disconnected cold population is locally green under `E-M5-ARTIFACT-CACHE-POPULATION-LOCAL-001`: exclusive cache-local download staging → strict existing immutable publication → staging cleanup → in-use lease, with failures propagating closed. Merging to `main` remains prohibited and SignPath is deferred. No Electron/startup, SSH, setting, tuple enablement, release publication, fallback, or default-path consumer exists. Legacy remains the production default.
+Current phase: Milestone 5 / Work Package 4 desktop resolver/cache — **In progress — 2026-07-15, Codex implementation owner: prove the locally green Windows cache-lock release correction on replacement all-six native CI before accepting cold-cache population**. Keep the correction independently evidenced and bounded; require the eventual startup caller to supply Orca's pre-`app.setName()` canonical user-data path; keep production manifest/resources, proxy, SSH/settings/tuples/release publication/default behavior absent. Real Apple/SignPath rehearsals remain an explicit late gate and are not on this package's critical path.
+Session checkpoint: **LOCAL GREEN / CI REQUIRED — 2026-07-15, Codex implementation owner** — `E-M5-ARTIFACT-CACHE-LOCK-RELEASE-LOCAL-001` implements and proves a 5 s/50 ms ownership-preserving retry only for Windows sharing-style `EPERM`/`EACCES`; displaced ownership stops without deletion and exhaustion fails closed. Focused/workflow, 339/339 non-full-size relay, 280/280 release-script, typecheck, lint, format, diff, and protected-resolver gates pass locally. The correction is not accepted until a new exact-head run replaces failing job 87548519537 across all six primary native jobs and adjacent checks. Merging to `main` remains prohibited and SignPath is deferred. No Electron/startup, SSH, setting, tuple enablement, release publication, fallback, or default-path consumer exists. Legacy remains the production default.
Primary design: [SSH relay GitHub Release plan](./2026-07-14-ssh-relay-github-release-plan.html)
Motivating issues: [#8450](https://github.com/stablyai/orca/issues/8450), [#1693](https://github.com/stablyai/orca/issues/1693) @@ -147,10 +147,11 @@ same change as the work it records. local gates, and all-six native Node 24 proof are green under E-M5-ARTIFACT-CACHE-ROOT-LOCAL-RED-001, E-M5-ARTIFACT-CACHE-ROOT-LOCAL-001, and E-M5-ARTIFACT-CACHE-ROOT-CI-001. No Electron or product caller exists. -- Active package: **In progress — 2026-07-15, Codex implementation owner** — audit the smallest - disconnected resolver/cache composition over explicit dependencies. The eventual startup caller, - not an SSH module, must supply `getCanonicalUserDataPath()` captured before `app.setName()`. - Keep proxy integration, SSH, mode, tuple, publication, and defaults separately gated. +- Active package: **In progress — 2026-07-15, Codex implementation owner** — prove the bounded + ownership-preserving cache-lock release correction on all six native clients before accepting + cold-cache population or auditing the next acquisition composition. The eventual startup caller, + not an SSH module, must supply `getCanonicalUserDataPath()` captured before `app.setName()`; keep + proxy integration, SSH, mode, tuple, publication, and defaults separately gated. - Completed Work Package 2 gate: target-native Windows source-signature reports from exact-head artifact jobs 87267322867 and 87267322870 were independently downloaded and matched to their identities and signing-stage reports under E-M3-WINDOWS-SOURCE-SIGNATURE-CI-001. PR Checks @@ -14603,6 +14604,84 @@ config/scripts/ssh-relay-runtime-workflow.test.mjs` — PASS, 4 files / 21 tests - Boundary: this corrects physical path identity only. It adds no network, Electron/startup caller, SSH, setting, fallback, tuple, release publication, or default behavior. +### E-M5-ARTIFACT-CACHE-LOCK-RELEASE-CI-RED-001 — repeated Windows active-owner release race + +- Date/owner: 2026-07-15, Codex implementation owner. +- Exact source/job: `3018977c113926addcbb97b5715242ed86d820d2`, artifact run + [29475848463](https://github.com/stablyai/orca/actions/runs/29475848463), Windows x64 job + [87548519537](https://github.com/stablyai/orca/actions/runs/29475848463/job/87548519537). +- Runner/runtime: GitHub-hosted Windows Server 2022 build 20348 x64, `windows-2022` image + `20260706.237.1`, Actions runner 2.335.1, Node 24.18.0. +- New-path result: both real cold-cache integration cases pass on the Windows client, including the + actual bounded downloader, Linux tar/Brotli and Windows ZIP strict publication, complete-tree + verification, physical-root cold→warm lookup, and in-use lease. +- Failure: the pre-existing `serializes concurrent writers and transfers ownership only after owner + release` test received `EPERM` renaming the owned lock to its release tombstone after the owner + handle closed. Test teardown then removed the root while the waiter was live, producing the + secondary `ENOENT` creating its next pending directory. Combined contract result: 1 failed, 489 + passed, 13 skipped across 76 files in 26.94 s. +- Repetition/decision: this matches `E-M5-COMPILED-TRUST-WINDOWS-CI-RED-001` (job 87532052082), whose + disposition required a separately evidenced correction if it repeated. It is no longer accepted + as a non-repeating runner event. Add a bounded retry only for Windows sharing-style `EPERM`/ + `EACCES`, re-check exact nonce/directory ownership before every retry, never delete a displaced + successor, retain unexpected-error propagation, and prove retry exhaustion is bounded. No test- + only relaxation or blind recursive deletion is allowed. +- Gate: do not accept `3018977c1` or begin the next acquisition package until a correction commit + passes replacement all-six native contracts and adjacent PR/E2E gates. + +### E-M5-ARTIFACT-CACHE-LOCK-RELEASE-LOCAL-RED-001 — bounded release retry is absent + +- Date/owner: 2026-07-15, Codex implementation owner. +- Command: `pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1 + src/main/ssh/ssh-relay-artifact-cache-lock-release.test.ts`. +- Result: expected FAIL, one failed suite / zero tests in 237 ms because the purpose-named + `ssh-relay-artifact-cache-lock-release` module does not exist. +- Fixed contract: 5 s/50 ms frozen bounds; retry only `EPERM`/`EACCES`; exact ownership recheck + before every retry; stop without deletion on displacement; fail closed at exact exhaustion; + already-absent path settles; unexpected errors propagate without retry. The test seam controls + time and filesystem operations, so exhaustion proof has no real delay or platform conditional. + +### E-M5-ARTIFACT-CACHE-LOCK-RELEASE-LOCAL-001 — bounded ownership-preserving release retry + +- Date/owner: 2026-07-15, Codex implementation owner. +- Implementation: lock release still closes its owned heartbeat handle before renaming the live + lock to its exact nonce-specific tombstone. Only rename `EPERM`/`EACCES` enters a frozen 50 ms, + 5 s retry policy. Each retry re-reads the exact directory identity and owner nonce immediately + before rename; displacement settles without deleting the successor. `ENOENT` is already + released, unexpected rename/ownership errors propagate, exact exhaustion fails closed, and only + a successfully renamed tombstone is recursively removed. There is no blind deletion of the live + lock path. +- Focused plus native-workflow oracle: `/usr/bin/time -l pnpm exec vitest run --config + config/vitest.config.ts --maxWorkers=1 + src/main/ssh/ssh-relay-artifact-cache-lock-release.test.ts + src/main/ssh/ssh-relay-artifact-cache-lock.test.ts + config/scripts/ssh-relay-runtime-workflow.test.mjs` — PASS, 3 files / 24 tests in 1.42 s Vitest, + 2.69 s wall, 132,562,944-byte maximum RSS. Seven purpose cases cover both sharing codes, exact + ownership rechecks, displaced-successor preservation, 100 controlled waits over the exact 5 s + budget, absent-path settlement, and unexpected rename/ownership error propagation. The workflow + oracle proves the purpose suite occurs exactly once in both POSIX and PowerShell native commands. +- Broader relay command: `/usr/bin/time -l pnpm exec vitest run --config + config/vitest.config.ts --maxWorkers=1 --silent=true src/main/ssh/ssh-relay-*.test.ts` — PASS, + 30 files / 339 tests; 3 declared full-size files/tests skipped; 21.01 s Vitest, 23.06 s wall, + 335,773,696-byte maximum RSS. An earlier quoted-glob invocation matched no files and exited 1; + the exact unquoted checklist command above was then run and passed. +- Release-script command: `/usr/bin/time -l pnpm exec vitest run --config + config/vitest.config.ts --maxWorkers=1 --silent=true + config/scripts/ssh-relay-runtime-*.test.mjs` — PASS, 50 files / 280 tests in 22.20 s Vitest, + 25.44 s wall, 194,150,400-byte maximum RSS. +- Repository gates: targeted `pnpm exec oxfmt --write` completes on five implementation/workflow + files in 428 ms; the post-format focused/workflow suite passes 24/24. `pnpm typecheck` and + `pnpm lint` pass; lint includes switch exhaustiveness, 41 reliability gates, no new max-lines + bypass, bundled skill-guide verification, 9,837 localization references and locale parity, and + zero localization coverage candidates. `git diff --check` passes and the protected + `ssh-remote-node-resolution.ts`/test pair has zero correction diff. Local Node 26.0.0 emits only + the expected Node-24 engine warning; exact Node 24 proof remains the native CI gate. +- Boundary/gate: the correction changes only cache-lock release settlement and native contract + execution. It adds no Electron/startup, network/proxy, SSH, setting, fallback classification, + tuple enablement, release publication, or default behavior. Require a replacement exact-head + all-six native run plus adjacent PR/E2E checks before accepting cold-cache population or starting + the next acquisition package. + ### E-M5-ARTIFACT-CACHE-POPULATION-LOCAL-001 — disconnected cold-cache population - Date/owner: 2026-07-15, Codex implementation owner. diff --git a/src/main/ssh/ssh-relay-artifact-cache-lock-lease.ts b/src/main/ssh/ssh-relay-artifact-cache-lock-lease.ts index 854e6de3c86..afbe89f8916 100644 --- a/src/main/ssh/ssh-relay-artifact-cache-lock-lease.ts +++ b/src/main/ssh/ssh-relay-artifact-cache-lock-lease.ts @@ -1,8 +1,11 @@ -import { lstat, rename, rm, type FileHandle } from 'node:fs/promises' +import { lstat, type FileHandle } from 'node:fs/promises' + +import { releaseSshRelayArtifactCacheLockPath } from './ssh-relay-artifact-cache-lock-release' import { readSshRelayArtifactCacheLockOwner, sameSshRelayArtifactCacheLockDirectory, + sshRelayArtifactCacheErrorCode, sshRelayArtifactCacheLockOwnerBytes, type SshRelayArtifactCacheLockDirectoryIdentity, type SshRelayArtifactCacheLockOwnerRecord @@ -72,14 +75,23 @@ class SshRelayArtifactCacheLockLease { }) } - private async assertPathOwnership(): Promise { - const metadata = await lstat(this.lockPath, { bigint: true }).catch(() => null) + private async ownsPath(): Promise { + const metadata = await lstat(this.lockPath, { bigint: true }).catch((error) => { + if (sshRelayArtifactCacheErrorCode(error) === 'ENOENT') { + return null + } + throw error + }) const current = await readSshRelayArtifactCacheLockOwner(this.lockPath, this.contentId) - if ( - !metadata?.isDirectory() || - !sameSshRelayArtifactCacheLockDirectory(this.directory, metadata) || - current?.token !== this.token - ) { + return Boolean( + metadata?.isDirectory() && + sameSshRelayArtifactCacheLockDirectory(this.directory, metadata) && + current?.token === this.token + ) + } + + private async assertPathOwnership(): Promise { + if (!(await this.ownsPath())) { throw new Error('SSH relay artifact cache lock ownership was displaced') } } @@ -118,9 +130,11 @@ class SshRelayArtifactCacheLockLease { // Windows cannot rename a directory containing an open file; ownership is final-checked first. await this.ownerHandle.close() - const tombstone = `${this.lockPath}.released-${this.token}` - await rename(this.lockPath, tombstone) - await rm(tombstone, { recursive: true, force: true }) + await releaseSshRelayArtifactCacheLockPath({ + lockPath: this.lockPath, + token: this.token, + checkOwnership: async () => ((await this.ownsPath()) ? 'owned' : 'displaced') + }) } } diff --git a/src/main/ssh/ssh-relay-artifact-cache-lock-release.test.ts b/src/main/ssh/ssh-relay-artifact-cache-lock-release.test.ts new file mode 100644 index 00000000000..4a3150982fe --- /dev/null +++ b/src/main/ssh/ssh-relay-artifact-cache-lock-release.test.ts @@ -0,0 +1,129 @@ +import { join } from 'node:path' + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + releaseSshRelayArtifactCacheLockPath, + SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS, + type SshRelayArtifactCacheLockReleaseOperations +} from './ssh-relay-artifact-cache-lock-release' + +function fileError(code: string): NodeJS.ErrnoException { + return Object.assign(new Error(`filesystem ${code}`), { code }) +} + +const lockPath = join('cache', 'locks', `${'a'.repeat(64)}.lock`) +const token = 'b'.repeat(32) +let now = 0 +const operations = { + rename: vi.fn(), + remove: vi.fn(), + wait: vi.fn(), + now: vi.fn() +} + +beforeEach(() => { + now = 0 + operations.rename.mockReset() + operations.remove.mockReset().mockResolvedValue(undefined) + operations.wait.mockReset().mockImplementation(async () => { + now += SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS.retryIntervalMs + }) + operations.now.mockReset().mockImplementation(() => now) +}) + +describe('SSH relay artifact cache lock release', () => { + it('pins a bounded retry policy below cancellation settlement', () => { + expect(SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS).toEqual({ + retryIntervalMs: 50, + retryTimeoutMs: 5_000 + }) + expect(Object.isFrozen(SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS)).toBe(true) + }) + + it('rechecks ownership before retrying transient Windows sharing failures', async () => { + const checkOwnership = vi.fn(async () => 'owned' as const) + operations.rename + .mockRejectedValueOnce(fileError('EPERM')) + .mockRejectedValueOnce(fileError('EACCES')) + .mockResolvedValueOnce(undefined) + + await expect( + releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations) + ).resolves.toBeUndefined() + + const tombstone = `${lockPath}.released-${token}` + expect(operations.rename).toHaveBeenCalledTimes(3) + expect(operations.rename).toHaveBeenNthCalledWith(1, lockPath, tombstone) + expect(checkOwnership).toHaveBeenCalledTimes(2) + expect(operations.wait).toHaveBeenCalledTimes(2) + expect(operations.remove).toHaveBeenCalledWith(tombstone) + }) + + it('stops without deleting when ownership is displaced during a sharing retry', async () => { + const checkOwnership = vi.fn(async () => 'displaced' as const) + operations.rename.mockRejectedValueOnce(fileError('EPERM')) + + await expect( + releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations) + ).resolves.toBeUndefined() + + expect(operations.rename).toHaveBeenCalledTimes(1) + expect(operations.wait).toHaveBeenCalledTimes(1) + expect(operations.remove).not.toHaveBeenCalled() + }) + + it('fails closed after the exact retry budget without deleting the owned lock', async () => { + const checkOwnership = vi.fn(async () => 'owned' as const) + operations.rename.mockRejectedValue(fileError('EPERM')) + + await expect( + releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations) + ).rejects.toThrow(/release|sharing|timeout|blocked/i) + + expect(now).toBe(SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS.retryTimeoutMs) + expect(operations.wait).toHaveBeenCalledTimes( + SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS.retryTimeoutMs / + SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS.retryIntervalMs + ) + expect(operations.remove).not.toHaveBeenCalled() + }) + + it('treats an already absent owned path as released', async () => { + const checkOwnership = vi.fn(async () => 'owned' as const) + operations.rename.mockRejectedValueOnce(fileError('ENOENT')) + + await expect( + releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations) + ).resolves.toBeUndefined() + expect(checkOwnership).not.toHaveBeenCalled() + expect(operations.wait).not.toHaveBeenCalled() + expect(operations.remove).not.toHaveBeenCalled() + }) + + it('propagates unexpected filesystem failures without retry or deletion', async () => { + const checkOwnership = vi.fn(async () => 'owned' as const) + operations.rename.mockRejectedValueOnce(fileError('EIO')) + + await expect( + releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations) + ).rejects.toThrow(/filesystem EIO/i) + expect(checkOwnership).not.toHaveBeenCalled() + expect(operations.wait).not.toHaveBeenCalled() + expect(operations.remove).not.toHaveBeenCalled() + }) + + it('propagates ownership-inspection errors instead of guessing displacement', async () => { + const checkOwnership = vi.fn(async () => { + throw fileError('EIO') + }) + operations.rename.mockRejectedValueOnce(fileError('EPERM')) + + await expect( + releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations) + ).rejects.toThrow(/filesystem EIO/i) + expect(operations.rename).toHaveBeenCalledTimes(1) + expect(operations.wait).toHaveBeenCalledTimes(1) + expect(operations.remove).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/ssh-relay-artifact-cache-lock-release.ts b/src/main/ssh/ssh-relay-artifact-cache-lock-release.ts new file mode 100644 index 00000000000..9a199c387d4 --- /dev/null +++ b/src/main/ssh/ssh-relay-artifact-cache-lock-release.ts @@ -0,0 +1,73 @@ +import { rename, rm } from 'node:fs/promises' + +import { sshRelayArtifactCacheErrorCode } from './ssh-relay-artifact-cache-lock-record' + +const RETRY_INTERVAL_MS = 50 +const RETRY_TIMEOUT_MS = 5_000 +const TOKEN = /^[0-9a-f]{32}$/ +const WINDOWS_SHARING_ERRORS = new Set(['EACCES', 'EPERM']) + +export const SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS = Object.freeze({ + retryIntervalMs: RETRY_INTERVAL_MS, + retryTimeoutMs: RETRY_TIMEOUT_MS +}) + +export type SshRelayArtifactCacheLockReleaseOperations = Readonly<{ + rename: (sourcePath: string, destinationPath: string) => Promise + remove: (path: string) => Promise + wait: () => Promise + now: () => number +}> + +const DEFAULT_OPERATIONS: SshRelayArtifactCacheLockReleaseOperations = Object.freeze({ + rename, + remove: async (path) => rm(path, { recursive: true, force: true }), + wait: async () => new Promise((resolveWait) => setTimeout(resolveWait, RETRY_INTERVAL_MS)), + now: Date.now +}) + +export async function releaseSshRelayArtifactCacheLockPath( + { + lockPath, + token, + checkOwnership + }: { + lockPath: string + token: string + checkOwnership: () => Promise<'owned' | 'displaced'> + }, + operations: SshRelayArtifactCacheLockReleaseOperations = DEFAULT_OPERATIONS +): Promise { + if (!TOKEN.test(token)) { + throw new Error('SSH relay artifact cache lock release token must be exact lowercase hex') + } + const tombstone = `${lockPath}.released-${token}` + const deadline = operations.now() + RETRY_TIMEOUT_MS + let retrying = false + while (true) { + if (retrying && (await checkOwnership()) === 'displaced') { + return + } + try { + await operations.rename(lockPath, tombstone) + break + } catch (error) { + const code = sshRelayArtifactCacheErrorCode(error) + if (code === 'ENOENT') { + return + } + if (!WINDOWS_SHARING_ERRORS.has(code ?? '')) { + throw error + } + if (operations.now() >= deadline) { + throw new Error('SSH relay artifact cache lock release remained blocked by file sharing', { + cause: error + }) + } + await operations.wait() + // Why: a sharing retry may proceed only while this exact nonce still owns this directory. + retrying = true + } + } + await operations.remove(tombstone) +}