mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
* fix(runtime): a device registry save never resurrects a device removed on disk Every DeviceRegistry save wrote orcad's whole in-memory list. A lastSeen flush (250 ms after any socket auth, and again at shutdown) could therefore write back a device an admin had just removed from orca-devices.json on the host, silently undoing the revoke; it also dropped rows another writer had added. Saves now re-read the file and write the in-memory change merged over it: rows this process knew of that are gone from disk stay gone (in memory too), rows added on disk are kept, and this save's own adds and removals apply. The write stays the secure-file atomic rename. The pairing-repair e2e logs the registry's device ids after the kill and after the reconnect. * fix(runtime): tombstone devices this process removed so a stale writer can't restore them Review of #35483: after removeDevice(a), a stale writer (an old orcad's flush during an SSH relaunch or upgrade overlap, an external tool) could write a back; the next save treated it as new on disk, adopted it and let it authenticate. DeviceRegistryWriter now remembers every id this process removed, recorded before the write so a failed load-time cleanup counts too, and never adopts those from disk. replaceRow reports false when the merge dropped the row, and a pending offer removed on disk is replaced instead of handed out.
218 lines
9.4 KiB
TypeScript
218 lines
9.4 KiB
TypeScript
// After host-side device removal/restart, re-pair over SSH and restore pane input automatically.
|
|
// A changed host key must refuse repair and retain the saved pairing.
|
|
import type { Page } from '@stablyai/playwright-test'
|
|
import { expect, test } from './helpers/orca-app'
|
|
import { readPairedPaneContent } from './helpers/paired-host-terminal'
|
|
import { reconnect } from './helpers/orcad-convert-flow'
|
|
import { startFreezableTcpProxy, type FreezableTcpProxy } from './helpers/freezable-tcp-proxy'
|
|
import {
|
|
echoMarker,
|
|
expectTerminalAnswers,
|
|
launchManagedWorkspace,
|
|
openTerminalTab,
|
|
skipUnlessManagedDockerHost,
|
|
withManagedHost
|
|
} from './helpers/orcad-managed-workspace'
|
|
|
|
skipUnlessManagedDockerHost()
|
|
test.skip(process.platform === 'win32', 'The Docker SSH host uses POSIX tooling')
|
|
|
|
const DEVICES = '/root/.orca/orca-devices.json'
|
|
const KEYPAIR = '/root/.orca/orca-e2ee-keypair.json'
|
|
const KILL_ORCAD = 'kill -TERM $(cat /root/.orca-remote/orcad-*/.orcad-pid)'
|
|
// Ids only: the rows also hold device tokens, which must stay out of CI logs.
|
|
const DEVICE_IDS =
|
|
`node -e 'console.log(JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"))` +
|
|
`.map((d)=>d.deviceId).join(","))' ${DEVICES}`
|
|
|
|
async function pairing(page: Page, environmentId: string) {
|
|
const environment = (await page.evaluate(() => window.api.runtimeEnvironments.list())).find(
|
|
(entry) => entry.id === environmentId
|
|
)
|
|
return {
|
|
revision: environment?.pairingRevision ?? environment?.createdAt ?? null,
|
|
deviceId: environment?.pairedDeviceId ?? null
|
|
}
|
|
}
|
|
|
|
function reconnectBanner(page: Page, tabId: string): Promise<string | null> {
|
|
return page.evaluate((id) => {
|
|
const pane = document.querySelector(`[data-terminal-tab-id="${id}"]`) ?? document
|
|
return (
|
|
pane
|
|
.querySelector('[data-terminal-remote-runtime-reconnect-banner]')
|
|
?.getAttribute('data-terminal-remote-runtime-reconnect-banner') ?? null
|
|
)
|
|
}, tabId)
|
|
}
|
|
|
|
test('a host that dropped this desktop is paired again over SSH and its pane answers', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
|
|
{}, testInfo) => {
|
|
test.setTimeout(15 * 60_000)
|
|
await withManagedHost(testInfo, async (fixture) => {
|
|
const { host } = fixture
|
|
const { page, seeded, environmentId } = await launchManagedWorkspace(fixture)
|
|
const tabId = await openTerminalTab(page, seeded.worktreeId)
|
|
await echoMarker(page, tabId, 'BEFORE_REVOKE')
|
|
const before = await pairing(page, environmentId)
|
|
expect(before.deviceId).toBeTruthy()
|
|
|
|
// Host side, as an admin would: remove this desktop's device, then restart the server.
|
|
host.exec!(
|
|
`node -e 'const f=process.argv[1];const fs=require("fs");` +
|
|
`fs.writeFileSync(f,JSON.stringify(JSON.parse(fs.readFileSync(f,"utf8"))` +
|
|
`.filter((d)=>d.deviceId!==process.argv[2])))' ${DEVICES} ${before.deviceId}`
|
|
)
|
|
host.exec!(KILL_ORCAD)
|
|
// Evidence for a revoked row the old server wrote back while stopping.
|
|
console.log('[pairing-repair] device ids after kill', host.exec!(DEVICE_IDS))
|
|
|
|
try {
|
|
await expectTerminalAnswers(page, tabId, 4 * 60_000)
|
|
} catch (error) {
|
|
console.log(
|
|
'[pairing-repair] evidence',
|
|
JSON.stringify({
|
|
banner: await reconnectBanner(page, tabId),
|
|
pane: (await readPairedPaneContent(page, tabId)).slice(-600),
|
|
devices: host.exec!(DEVICE_IDS)
|
|
})
|
|
)
|
|
await page.screenshot({ path: testInfo.outputPath('after-revoke.png') })
|
|
throw error
|
|
}
|
|
const after = await pairing(page, environmentId)
|
|
console.log('[pairing-repair] device ids after reconnect', host.exec!(DEVICE_IDS), {
|
|
revoked: before.deviceId,
|
|
paired: after.deviceId
|
|
})
|
|
expect(after.revision).not.toBe(before.revision)
|
|
expect(after.deviceId).not.toBe(before.deviceId)
|
|
await expect.poll(() => reconnectBanner(page, tabId)).toBeNull()
|
|
await page.screenshot({ path: testInfo.outputPath('repaired.png') })
|
|
})
|
|
})
|
|
|
|
test('a server that now holds another host key is never paired, and connecting says so', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
|
|
{}, testInfo) => {
|
|
test.setTimeout(15 * 60_000)
|
|
await withManagedHost(testInfo, async (fixture) => {
|
|
const { host } = fixture
|
|
const { page, seeded, environmentId } = await launchManagedWorkspace(fixture)
|
|
const tabId = await openTerminalTab(page, seeded.worktreeId)
|
|
await echoMarker(page, tabId, 'BEFORE_RESET')
|
|
const before = await pairing(page, environmentId)
|
|
|
|
// The server's identity is reset: a new key and no devices, as a wiped profile would have.
|
|
host.exec!(`rm -f ${KEYPAIR} ${DEVICES} && ${KILL_ORCAD}`)
|
|
|
|
// The first connect may find the server down and start it; the next one meets the new key.
|
|
let connected = ''
|
|
await expect
|
|
.poll(
|
|
async () => {
|
|
// The helper throws when the connect is refused; the refusal is the answer here.
|
|
connected = await reconnect(page, seeded.targetId).catch((error: unknown) =>
|
|
String(error)
|
|
)
|
|
return connected
|
|
},
|
|
{ timeout: 3 * 60_000, intervals: [1_000, 5_000] }
|
|
)
|
|
.toMatch(/orcad_identity_mismatch/)
|
|
expect(connected).not.toMatch(/pairing_repair/)
|
|
expect(await pairing(page, environmentId)).toEqual(before)
|
|
// Nothing re-pairs in the background either.
|
|
await page.waitForTimeout(20_000)
|
|
expect(await pairing(page, environmentId)).toEqual(before)
|
|
await page.screenshot({ path: testInfo.outputPath('foreign.png') })
|
|
})
|
|
})
|
|
|
|
// Restart behind the network cut so the first client observation is the refusal from the deliberately staged offer.
|
|
const CAPTURE_LAUNCH =
|
|
'f=$(ls /root/.orca-remote/orcad-*/.orcad-pid) && pid=$(cat "$f") && echo "$pid" > /tmp/orcad-old-pid && dirname "$f" > /tmp/orcad-dir && ' +
|
|
"tr '\\0' '\\n' < /proc/$pid/cmdline > /tmp/orcad-cmd && tr '\\0' '\\n' < /proc/$pid/environ > /tmp/orcad-env"
|
|
const RELAUNCH =
|
|
'd=$(cat /tmp/orcad-dir) && cd "$d" && : > .orcad-readiness && ' +
|
|
'mapfile -t c < /tmp/orcad-cmd && mapfile -t e < /tmp/orcad-env && ' +
|
|
'{ env -i "${e[@]}" nohup "${c[@]}" > .orcad-readiness 2>/dev/null < /dev/null & echo $! > .orcad-pid; }'
|
|
const WITHDRAW_OFFER =
|
|
`node -e 'const fs=require("fs");const f=process.argv[1];fs.writeFileSync(f,fs.readFileSync(f,"utf8")` +
|
|
`.split("\\n").map((l)=>{try{const j=JSON.parse(l);if(j&&j.pairing){j.pairing={available:false,` +
|
|
`reason:"disabled_by_operator",guidance:"no offer in this test"}}return JSON.stringify(j)}catch{return l}})` +
|
|
`.join("\\n"))' "$(cat /tmp/orcad-dir)/.orcad-readiness"`
|
|
|
|
test('a failed re-pair says why, and the pane stops reconnecting with that verdict', async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
|
|
{}, testInfo) => {
|
|
test.setTimeout(15 * 60_000)
|
|
const opened: FreezableTcpProxy[] = []
|
|
try {
|
|
await withManagedHost(testInfo, async (fixture) => {
|
|
const { host } = fixture
|
|
const proxy = await startFreezableTcpProxy(host.input.host, host.input.port ?? 22)
|
|
opened.push(proxy)
|
|
host.input = { ...host.input, port: proxy.port }
|
|
const { page, seeded, environmentId } = await launchManagedWorkspace(fixture)
|
|
const tabId = await openTerminalTab(page, seeded.worktreeId)
|
|
await echoMarker(page, tabId, 'BEFORE_REVOKE')
|
|
const before = await pairing(page, environmentId)
|
|
|
|
host.exec!(CAPTURE_LAUNCH)
|
|
proxy.cut('reset')
|
|
host.exec!(
|
|
`node -e 'const f=process.argv[1];const fs=require("fs");` +
|
|
`fs.writeFileSync(f,JSON.stringify(JSON.parse(fs.readFileSync(f,"utf8"))` +
|
|
`.filter((d)=>d.deviceId!==process.argv[2])))' ${DEVICES} ${before.deviceId}`
|
|
)
|
|
host.exec!(
|
|
`${KILL_ORCAD}; while kill -0 $(cat /tmp/orcad-old-pid) 2>/dev/null; do sleep 0.2; done; true`
|
|
)
|
|
host.exec!(RELAUNCH)
|
|
await expect
|
|
.poll(
|
|
() =>
|
|
host.exec!('test -s "$(cat /tmp/orcad-dir)/.orcad-readiness" && echo ready || true'),
|
|
{
|
|
timeout: 60_000
|
|
}
|
|
)
|
|
.toContain('ready')
|
|
host.exec!(WITHDRAW_OFFER)
|
|
proxy.restore()
|
|
|
|
try {
|
|
// The pane stops "Reconnecting" and says why, rather than a bare "disconnected".
|
|
await expect
|
|
.poll(() => reconnectBanner(page, tabId), { timeout: 4 * 60_000 })
|
|
.toBe('pairing-refused')
|
|
} catch (error) {
|
|
console.log(
|
|
'[pairing-repair] evidence',
|
|
JSON.stringify({
|
|
banner: await reconnectBanner(page, tabId),
|
|
status: await page.evaluate(
|
|
(id) => window.__store?.getState().runtimeStatusByEnvironmentId.get(id)?.snapshot,
|
|
environmentId
|
|
),
|
|
ssh: await page.evaluate(
|
|
(id) => window.__store?.getState().sshConnectionStates.get(id),
|
|
seeded.targetId
|
|
),
|
|
readiness: host.exec!('cat "$(cat /tmp/orcad-dir)/.orcad-readiness" | head -c 600')
|
|
})
|
|
)
|
|
await page.screenshot({ path: testInfo.outputPath('repair-failed-evidence.png') })
|
|
throw error
|
|
}
|
|
expect(await page.evaluate(() => document.body.innerText)).toMatch(
|
|
/couldn.t pair with it again over SSH/
|
|
)
|
|
expect(await pairing(page, environmentId)).toEqual(before)
|
|
await page.screenshot({ path: testInfo.outputPath('repair-failed.png') })
|
|
})
|
|
} finally {
|
|
await Promise.all(opened.map((proxy) => proxy.close()))
|
|
}
|
|
})
|