fix: settle Windows relay cache locks

Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Jinwoo-H
2026-07-16 02:25:46 -04:00
co-authored by Orca
parent 3018977c11
commit bd240049ff
7 changed files with 337 additions and 20 deletions
@@ -264,6 +264,7 @@ jobs:
src/main/ssh/ssh-relay-artifact-selector.test.ts \
src/main/ssh/ssh-relay-artifact-download.test.ts \
src/main/ssh/ssh-relay-artifact-extraction.test.ts \
src/main/ssh/ssh-relay-artifact-cache-lock-release.test.ts \
src/main/ssh/ssh-relay-artifact-cache-lock.test.ts \
src/main/ssh/ssh-relay-artifact-cache-entry.test.ts \
src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts \
@@ -806,6 +807,7 @@ jobs:
src/main/ssh/ssh-relay-artifact-selector.test.ts `
src/main/ssh/ssh-relay-artifact-download.test.ts `
src/main/ssh/ssh-relay-artifact-extraction.test.ts `
src/main/ssh/ssh-relay-artifact-cache-lock-release.test.ts `
src/main/ssh/ssh-relay-artifact-cache-lock.test.ts `
src/main/ssh/ssh-relay-artifact-cache-entry.test.ts `
src/main/ssh/ssh-relay-artifact-cache-in-use-lease.test.ts `
@@ -122,6 +122,7 @@ describe('SSH relay runtime artifact workflow', () => {
'ssh-relay-artifact-selector.test.ts',
'ssh-relay-artifact-download.test.ts',
'ssh-relay-artifact-extraction.test.ts',
'ssh-relay-artifact-cache-lock-release.test.ts',
'ssh-relay-artifact-cache-lock.test.ts',
'ssh-relay-artifact-cache-entry.test.ts',
'ssh-relay-artifact-cache-in-use-lease.test.ts',
@@ -9,9 +9,10 @@ keeps commands, hashes, runner identities, timings, and failure details.
A checked box means the work has evidence in the detailed ledger. Design approval alone does not
complete a box.
Active checkpoint: **Milestone 5 / Work Package 4 desktop resolver/cache — disconnected cold-cache
population locally green and wired; commit/all-six native proof next, 2026-07-15, Codex
implementation owner.** `E-M5-ARTIFACT-CACHE-RESOLUTION-CI-001` closes warm-cache commit
Active checkpoint: **Milestone 5 / Work Package 4 desktop resolver/cache — prove the locally green
Windows active-owner cache-lock release correction on replacement all-six native CI before
accepting cold-cache population, 2026-07-15, Codex implementation owner.**
`E-M5-ARTIFACT-CACHE-RESOLUTION-CI-001` closes warm-cache commit
`22031fa68` on all six native clients, both Linux supplements, Windows x64 baseline, PR Checks,
Golden E2E, and computer-use; Windows arm64 remains the expected build-26200 rejection against 26100. `E-M5-ARTIFACT-CACHE-RESOLUTION-LOCAL-001` passes 9/9 purpose tests,
16/16 focused+workflow-oracle tests, 323/323 non-full-size SSH relay tests, 280/280 release-script
@@ -38,6 +39,24 @@ implementation now canonicalizes lease identity/locking/recency to the existing
`E-M5-ARTIFACT-CACHE-PHYSICAL-ROOT-CORRECTION-LOCAL-001`: 7/7 correction tests and 21/21 focused+
workflow-oracle tests pass while missing/misplaced entries remain rejected. Broader exact-head proof
is still required before checkpointing.
Exact-head cold-cache artifact run 29475848463 then repeated the prior Windows x64 active-owner
release race in job 87548519537: the new real Linux/Windows cold→warm integration cases pass, but
lock tombstone rename receives `EPERM` and the live waiter later receives teardown `ENOENT`
(`E-M5-ARTIFACT-CACHE-LOCK-RELEASE-CI-RED-001`). The prior ledger explicitly required correction if
this recurred. The bounded sharing-error correction below is locally green, but replacement all-six
native proof remains required; do not advance to acquisition, Electron/startup, SSH, settings,
fallback, tuple, publication, or default behavior.
`E-M5-ARTIFACT-CACHE-LOCK-RELEASE-LOCAL-RED-001` records the expected missing-module failure for the
fixed bounded retry/displacement/exhaustion contract before implementation.
`E-M5-ARTIFACT-CACHE-LOCK-RELEASE-LOCAL-001` is now locally green: only `EPERM`/`EACCES` rename
failures retry for at most 5 s at 50 ms intervals; every retry rechecks exact directory/nonce
ownership, displacement preserves the successor, absent paths settle, and unexpected or exhausted
failures propagate closed. Focused plus workflow-oracle proof passes 24/24, broader non-full-size
relay proof passes 339/339 with the three declared full-size skips, release scripts pass 280/280,
and typecheck, lint, format, diff, and protected-resolver gates pass. A replacement exact-head
all-six native run plus adjacent PR/E2E checks is still mandatory before this correction or the
cold-cache population package is accepted; do not advance to acquisition, Electron/startup, SSH,
settings, fallback, tuple, publication, or default behavior yet.
`E-M5-ARTIFACT-CACHE-ROOT-CI-001` closes the pure cache-root contract at exact head `aefcaa9a9`:
all six primary native Node 24 jobs, both Linux supplements, Windows x64 baseline, PR Checks, Golden
E2E, and computer-use pass; Windows arm64 build 26200 remains correctly gated against 26100. The
@@ -8,8 +8,8 @@ work; keep exact commands, runner identities, hashes, metrics, and residual gaps
Date created: 2026-07-14<br>
Last updated: 2026-07-15<br>
Current phase: Milestone 5 / Work Package 4 desktop resolver/cache — **In progress — 2026-07-15, Codex implementation owner: prove disconnected cold-cache population**. Require the eventual startup caller to supply Orca's pre-`app.setName()` canonical user-data path; keep filesystem orchestration outside import time and production manifest/resources, proxy, SSH/settings/tuples/release publication/default behavior absent. Real Apple/SignPath rehearsals remain an explicit late gate and are not on this package's critical path.<br>
Session checkpoint: **In progress — 2026-07-15, Codex implementation owner** — `E-M5-ARTIFACT-CACHE-RESOLUTION-CI-001` closes warm-cache commit `22031fa686d2a7c96e687bc02c16fdb08bc987d7` on all six primary native clients, both Linux supplements, Windows x64 baseline, PR Checks, Golden E2E, and computer-use; Windows arm64 remains the expected build-26200 rejection against 26100. The disconnected cold population is locally green under `E-M5-ARTIFACT-CACHE-POPULATION-LOCAL-001`: exclusive cache-local download staging → strict existing immutable publication → staging cleanup → in-use lease, with failures propagating closed. Merging to `main` remains prohibited and SignPath is deferred. No Electron/startup, SSH, setting, tuple enablement, release publication, fallback, or default-path consumer exists. Legacy remains the production default.<br>
Current phase: Milestone 5 / Work Package 4 desktop resolver/cache — **In progress — 2026-07-15, Codex implementation owner: prove the locally green Windows cache-lock release correction on replacement all-six native CI before accepting cold-cache population**. Keep the correction independently evidenced and bounded; require the eventual startup caller to supply Orca's pre-`app.setName()` canonical user-data path; keep production manifest/resources, proxy, SSH/settings/tuples/release publication/default behavior absent. Real Apple/SignPath rehearsals remain an explicit late gate and are not on this package's critical path.<br>
Session checkpoint: **LOCAL GREEN / CI REQUIRED — 2026-07-15, Codex implementation owner** — `E-M5-ARTIFACT-CACHE-LOCK-RELEASE-LOCAL-001` implements and proves a 5 s/50 ms ownership-preserving retry only for Windows sharing-style `EPERM`/`EACCES`; displaced ownership stops without deletion and exhaustion fails closed. Focused/workflow, 339/339 non-full-size relay, 280/280 release-script, typecheck, lint, format, diff, and protected-resolver gates pass locally. The correction is not accepted until a new exact-head run replaces failing job 87548519537 across all six primary native jobs and adjacent checks. Merging to `main` remains prohibited and SignPath is deferred. No Electron/startup, SSH, setting, tuple enablement, release publication, fallback, or default-path consumer exists. Legacy remains the production default.<br>
Primary design: [SSH relay GitHub Release plan](./2026-07-14-ssh-relay-github-release-plan.html)<br>
Motivating issues: [#8450](https://github.com/stablyai/orca/issues/8450), [#1693](https://github.com/stablyai/orca/issues/1693)
@@ -147,10 +147,11 @@ same change as the work it records.
local gates, and all-six native Node 24 proof are green under
E-M5-ARTIFACT-CACHE-ROOT-LOCAL-RED-001, E-M5-ARTIFACT-CACHE-ROOT-LOCAL-001, and
E-M5-ARTIFACT-CACHE-ROOT-CI-001. No Electron or product caller exists.
- Active package: **In progress — 2026-07-15, Codex implementation owner** — audit the smallest
disconnected resolver/cache composition over explicit dependencies. The eventual startup caller,
not an SSH module, must supply `getCanonicalUserDataPath()` captured before `app.setName()`.
Keep proxy integration, SSH, mode, tuple, publication, and defaults separately gated.
- Active package: **In progress — 2026-07-15, Codex implementation owner** — prove the bounded
ownership-preserving cache-lock release correction on all six native clients before accepting
cold-cache population or auditing the next acquisition composition. The eventual startup caller,
not an SSH module, must supply `getCanonicalUserDataPath()` captured before `app.setName()`; keep
proxy integration, SSH, mode, tuple, publication, and defaults separately gated.
- Completed Work Package 2 gate: target-native Windows source-signature reports from exact-head
artifact jobs 87267322867 and 87267322870 were independently downloaded and matched to their
identities and signing-stage reports under E-M3-WINDOWS-SOURCE-SIGNATURE-CI-001. PR Checks
@@ -14603,6 +14604,84 @@ config/scripts/ssh-relay-runtime-workflow.test.mjs` — PASS, 4 files / 21 tests
- Boundary: this corrects physical path identity only. It adds no network, Electron/startup caller,
SSH, setting, fallback, tuple, release publication, or default behavior.
### E-M5-ARTIFACT-CACHE-LOCK-RELEASE-CI-RED-001 — repeated Windows active-owner release race
- Date/owner: 2026-07-15, Codex implementation owner.
- Exact source/job: `3018977c113926addcbb97b5715242ed86d820d2`, artifact run
[29475848463](https://github.com/stablyai/orca/actions/runs/29475848463), Windows x64 job
[87548519537](https://github.com/stablyai/orca/actions/runs/29475848463/job/87548519537).
- Runner/runtime: GitHub-hosted Windows Server 2022 build 20348 x64, `windows-2022` image
`20260706.237.1`, Actions runner 2.335.1, Node 24.18.0.
- New-path result: both real cold-cache integration cases pass on the Windows client, including the
actual bounded downloader, Linux tar/Brotli and Windows ZIP strict publication, complete-tree
verification, physical-root cold→warm lookup, and in-use lease.
- Failure: the pre-existing `serializes concurrent writers and transfers ownership only after owner
release` test received `EPERM` renaming the owned lock to its release tombstone after the owner
handle closed. Test teardown then removed the root while the waiter was live, producing the
secondary `ENOENT` creating its next pending directory. Combined contract result: 1 failed, 489
passed, 13 skipped across 76 files in 26.94 s.
- Repetition/decision: this matches `E-M5-COMPILED-TRUST-WINDOWS-CI-RED-001` (job 87532052082), whose
disposition required a separately evidenced correction if it repeated. It is no longer accepted
as a non-repeating runner event. Add a bounded retry only for Windows sharing-style `EPERM`/
`EACCES`, re-check exact nonce/directory ownership before every retry, never delete a displaced
successor, retain unexpected-error propagation, and prove retry exhaustion is bounded. No test-
only relaxation or blind recursive deletion is allowed.
- Gate: do not accept `3018977c1` or begin the next acquisition package until a correction commit
passes replacement all-six native contracts and adjacent PR/E2E gates.
### E-M5-ARTIFACT-CACHE-LOCK-RELEASE-LOCAL-RED-001 — bounded release retry is absent
- Date/owner: 2026-07-15, Codex implementation owner.
- Command: `pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1
src/main/ssh/ssh-relay-artifact-cache-lock-release.test.ts`.
- Result: expected FAIL, one failed suite / zero tests in 237 ms because the purpose-named
`ssh-relay-artifact-cache-lock-release` module does not exist.
- Fixed contract: 5 s/50 ms frozen bounds; retry only `EPERM`/`EACCES`; exact ownership recheck
before every retry; stop without deletion on displacement; fail closed at exact exhaustion;
already-absent path settles; unexpected errors propagate without retry. The test seam controls
time and filesystem operations, so exhaustion proof has no real delay or platform conditional.
### E-M5-ARTIFACT-CACHE-LOCK-RELEASE-LOCAL-001 — bounded ownership-preserving release retry
- Date/owner: 2026-07-15, Codex implementation owner.
- Implementation: lock release still closes its owned heartbeat handle before renaming the live
lock to its exact nonce-specific tombstone. Only rename `EPERM`/`EACCES` enters a frozen 50 ms,
5 s retry policy. Each retry re-reads the exact directory identity and owner nonce immediately
before rename; displacement settles without deleting the successor. `ENOENT` is already
released, unexpected rename/ownership errors propagate, exact exhaustion fails closed, and only
a successfully renamed tombstone is recursively removed. There is no blind deletion of the live
lock path.
- Focused plus native-workflow oracle: `/usr/bin/time -l pnpm exec vitest run --config
config/vitest.config.ts --maxWorkers=1
src/main/ssh/ssh-relay-artifact-cache-lock-release.test.ts
src/main/ssh/ssh-relay-artifact-cache-lock.test.ts
config/scripts/ssh-relay-runtime-workflow.test.mjs` — PASS, 3 files / 24 tests in 1.42 s Vitest,
2.69 s wall, 132,562,944-byte maximum RSS. Seven purpose cases cover both sharing codes, exact
ownership rechecks, displaced-successor preservation, 100 controlled waits over the exact 5 s
budget, absent-path settlement, and unexpected rename/ownership error propagation. The workflow
oracle proves the purpose suite occurs exactly once in both POSIX and PowerShell native commands.
- Broader relay command: `/usr/bin/time -l pnpm exec vitest run --config
config/vitest.config.ts --maxWorkers=1 --silent=true src/main/ssh/ssh-relay-*.test.ts` — PASS,
30 files / 339 tests; 3 declared full-size files/tests skipped; 21.01 s Vitest, 23.06 s wall,
335,773,696-byte maximum RSS. An earlier quoted-glob invocation matched no files and exited 1;
the exact unquoted checklist command above was then run and passed.
- Release-script command: `/usr/bin/time -l pnpm exec vitest run --config
config/vitest.config.ts --maxWorkers=1 --silent=true
config/scripts/ssh-relay-runtime-*.test.mjs` — PASS, 50 files / 280 tests in 22.20 s Vitest,
25.44 s wall, 194,150,400-byte maximum RSS.
- Repository gates: targeted `pnpm exec oxfmt --write` completes on five implementation/workflow
files in 428 ms; the post-format focused/workflow suite passes 24/24. `pnpm typecheck` and
`pnpm lint` pass; lint includes switch exhaustiveness, 41 reliability gates, no new max-lines
bypass, bundled skill-guide verification, 9,837 localization references and locale parity, and
zero localization coverage candidates. `git diff --check` passes and the protected
`ssh-remote-node-resolution.ts`/test pair has zero correction diff. Local Node 26.0.0 emits only
the expected Node-24 engine warning; exact Node 24 proof remains the native CI gate.
- Boundary/gate: the correction changes only cache-lock release settlement and native contract
execution. It adds no Electron/startup, network/proxy, SSH, setting, fallback classification,
tuple enablement, release publication, or default behavior. Require a replacement exact-head
all-six native run plus adjacent PR/E2E checks before accepting cold-cache population or starting
the next acquisition package.
### E-M5-ARTIFACT-CACHE-POPULATION-LOCAL-001 — disconnected cold-cache population
- Date/owner: 2026-07-15, Codex implementation owner.
@@ -1,8 +1,11 @@
import { lstat, rename, rm, type FileHandle } from 'node:fs/promises'
import { lstat, type FileHandle } from 'node:fs/promises'
import { releaseSshRelayArtifactCacheLockPath } from './ssh-relay-artifact-cache-lock-release'
import {
readSshRelayArtifactCacheLockOwner,
sameSshRelayArtifactCacheLockDirectory,
sshRelayArtifactCacheErrorCode,
sshRelayArtifactCacheLockOwnerBytes,
type SshRelayArtifactCacheLockDirectoryIdentity,
type SshRelayArtifactCacheLockOwnerRecord
@@ -72,14 +75,23 @@ class SshRelayArtifactCacheLockLease {
})
}
private async assertPathOwnership(): Promise<void> {
const metadata = await lstat(this.lockPath, { bigint: true }).catch(() => null)
private async ownsPath(): Promise<boolean> {
const metadata = await lstat(this.lockPath, { bigint: true }).catch((error) => {
if (sshRelayArtifactCacheErrorCode(error) === 'ENOENT') {
return null
}
throw error
})
const current = await readSshRelayArtifactCacheLockOwner(this.lockPath, this.contentId)
if (
!metadata?.isDirectory() ||
!sameSshRelayArtifactCacheLockDirectory(this.directory, metadata) ||
current?.token !== this.token
) {
return Boolean(
metadata?.isDirectory() &&
sameSshRelayArtifactCacheLockDirectory(this.directory, metadata) &&
current?.token === this.token
)
}
private async assertPathOwnership(): Promise<void> {
if (!(await this.ownsPath())) {
throw new Error('SSH relay artifact cache lock ownership was displaced')
}
}
@@ -118,9 +130,11 @@ class SshRelayArtifactCacheLockLease {
// Windows cannot rename a directory containing an open file; ownership is final-checked first.
await this.ownerHandle.close()
const tombstone = `${this.lockPath}.released-${this.token}`
await rename(this.lockPath, tombstone)
await rm(tombstone, { recursive: true, force: true })
await releaseSshRelayArtifactCacheLockPath({
lockPath: this.lockPath,
token: this.token,
checkOwnership: async () => ((await this.ownsPath()) ? 'owned' : 'displaced')
})
}
}
@@ -0,0 +1,129 @@
import { join } from 'node:path'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import {
releaseSshRelayArtifactCacheLockPath,
SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS,
type SshRelayArtifactCacheLockReleaseOperations
} from './ssh-relay-artifact-cache-lock-release'
function fileError(code: string): NodeJS.ErrnoException {
return Object.assign(new Error(`filesystem ${code}`), { code })
}
const lockPath = join('cache', 'locks', `${'a'.repeat(64)}.lock`)
const token = 'b'.repeat(32)
let now = 0
const operations = {
rename: vi.fn<SshRelayArtifactCacheLockReleaseOperations['rename']>(),
remove: vi.fn<SshRelayArtifactCacheLockReleaseOperations['remove']>(),
wait: vi.fn<SshRelayArtifactCacheLockReleaseOperations['wait']>(),
now: vi.fn<SshRelayArtifactCacheLockReleaseOperations['now']>()
}
beforeEach(() => {
now = 0
operations.rename.mockReset()
operations.remove.mockReset().mockResolvedValue(undefined)
operations.wait.mockReset().mockImplementation(async () => {
now += SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS.retryIntervalMs
})
operations.now.mockReset().mockImplementation(() => now)
})
describe('SSH relay artifact cache lock release', () => {
it('pins a bounded retry policy below cancellation settlement', () => {
expect(SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS).toEqual({
retryIntervalMs: 50,
retryTimeoutMs: 5_000
})
expect(Object.isFrozen(SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS)).toBe(true)
})
it('rechecks ownership before retrying transient Windows sharing failures', async () => {
const checkOwnership = vi.fn(async () => 'owned' as const)
operations.rename
.mockRejectedValueOnce(fileError('EPERM'))
.mockRejectedValueOnce(fileError('EACCES'))
.mockResolvedValueOnce(undefined)
await expect(
releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations)
).resolves.toBeUndefined()
const tombstone = `${lockPath}.released-${token}`
expect(operations.rename).toHaveBeenCalledTimes(3)
expect(operations.rename).toHaveBeenNthCalledWith(1, lockPath, tombstone)
expect(checkOwnership).toHaveBeenCalledTimes(2)
expect(operations.wait).toHaveBeenCalledTimes(2)
expect(operations.remove).toHaveBeenCalledWith(tombstone)
})
it('stops without deleting when ownership is displaced during a sharing retry', async () => {
const checkOwnership = vi.fn(async () => 'displaced' as const)
operations.rename.mockRejectedValueOnce(fileError('EPERM'))
await expect(
releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations)
).resolves.toBeUndefined()
expect(operations.rename).toHaveBeenCalledTimes(1)
expect(operations.wait).toHaveBeenCalledTimes(1)
expect(operations.remove).not.toHaveBeenCalled()
})
it('fails closed after the exact retry budget without deleting the owned lock', async () => {
const checkOwnership = vi.fn(async () => 'owned' as const)
operations.rename.mockRejectedValue(fileError('EPERM'))
await expect(
releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations)
).rejects.toThrow(/release|sharing|timeout|blocked/i)
expect(now).toBe(SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS.retryTimeoutMs)
expect(operations.wait).toHaveBeenCalledTimes(
SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS.retryTimeoutMs /
SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS.retryIntervalMs
)
expect(operations.remove).not.toHaveBeenCalled()
})
it('treats an already absent owned path as released', async () => {
const checkOwnership = vi.fn(async () => 'owned' as const)
operations.rename.mockRejectedValueOnce(fileError('ENOENT'))
await expect(
releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations)
).resolves.toBeUndefined()
expect(checkOwnership).not.toHaveBeenCalled()
expect(operations.wait).not.toHaveBeenCalled()
expect(operations.remove).not.toHaveBeenCalled()
})
it('propagates unexpected filesystem failures without retry or deletion', async () => {
const checkOwnership = vi.fn(async () => 'owned' as const)
operations.rename.mockRejectedValueOnce(fileError('EIO'))
await expect(
releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations)
).rejects.toThrow(/filesystem EIO/i)
expect(checkOwnership).not.toHaveBeenCalled()
expect(operations.wait).not.toHaveBeenCalled()
expect(operations.remove).not.toHaveBeenCalled()
})
it('propagates ownership-inspection errors instead of guessing displacement', async () => {
const checkOwnership = vi.fn(async () => {
throw fileError('EIO')
})
operations.rename.mockRejectedValueOnce(fileError('EPERM'))
await expect(
releaseSshRelayArtifactCacheLockPath({ lockPath, token, checkOwnership }, operations)
).rejects.toThrow(/filesystem EIO/i)
expect(operations.rename).toHaveBeenCalledTimes(1)
expect(operations.wait).toHaveBeenCalledTimes(1)
expect(operations.remove).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,73 @@
import { rename, rm } from 'node:fs/promises'
import { sshRelayArtifactCacheErrorCode } from './ssh-relay-artifact-cache-lock-record'
const RETRY_INTERVAL_MS = 50
const RETRY_TIMEOUT_MS = 5_000
const TOKEN = /^[0-9a-f]{32}$/
const WINDOWS_SHARING_ERRORS = new Set(['EACCES', 'EPERM'])
export const SSH_RELAY_ARTIFACT_CACHE_LOCK_RELEASE_LIMITS = Object.freeze({
retryIntervalMs: RETRY_INTERVAL_MS,
retryTimeoutMs: RETRY_TIMEOUT_MS
})
export type SshRelayArtifactCacheLockReleaseOperations = Readonly<{
rename: (sourcePath: string, destinationPath: string) => Promise<void>
remove: (path: string) => Promise<void>
wait: () => Promise<void>
now: () => number
}>
const DEFAULT_OPERATIONS: SshRelayArtifactCacheLockReleaseOperations = Object.freeze({
rename,
remove: async (path) => rm(path, { recursive: true, force: true }),
wait: async () => new Promise<void>((resolveWait) => setTimeout(resolveWait, RETRY_INTERVAL_MS)),
now: Date.now
})
export async function releaseSshRelayArtifactCacheLockPath(
{
lockPath,
token,
checkOwnership
}: {
lockPath: string
token: string
checkOwnership: () => Promise<'owned' | 'displaced'>
},
operations: SshRelayArtifactCacheLockReleaseOperations = DEFAULT_OPERATIONS
): Promise<void> {
if (!TOKEN.test(token)) {
throw new Error('SSH relay artifact cache lock release token must be exact lowercase hex')
}
const tombstone = `${lockPath}.released-${token}`
const deadline = operations.now() + RETRY_TIMEOUT_MS
let retrying = false
while (true) {
if (retrying && (await checkOwnership()) === 'displaced') {
return
}
try {
await operations.rename(lockPath, tombstone)
break
} catch (error) {
const code = sshRelayArtifactCacheErrorCode(error)
if (code === 'ENOENT') {
return
}
if (!WINDOWS_SHARING_ERRORS.has(code ?? '')) {
throw error
}
if (operations.now() >= deadline) {
throw new Error('SSH relay artifact cache lock release remained blocked by file sharing', {
cause: error
})
}
await operations.wait()
// Why: a sharing retry may proceed only while this exact nonce still owns this directory.
retrying = true
}
}
await operations.remove(tombstone)
}