mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 03:15:01 +00:00
fix(orchestration): task-update on this Orca's own Task applies here, by the local rule
A Task placed on a paired server is this Orca's; its update goes through the ordinary owner, which refuses it while the worker there holds it, exactly as for a local worker. Stop first: an unanswered stop settles the Task here at once. Nothing is forwarded to the server.
This commit is contained in:
@@ -418,38 +418,6 @@ describe('what the coordinating Orca owes a paired server', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('#8 forwarded taskUpdate', () => {
|
||||
it('sends a Task update to the server running its attempt instead of applying it here', async () => {
|
||||
startedOnB()
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS })
|
||||
answers['orchestration.taskUpdate'] = [
|
||||
{ result: { task: { id: 'task_t', status: 'completed' } } }
|
||||
]
|
||||
const response = await dispatcher.dispatch({
|
||||
id: 'update',
|
||||
authToken: 'local',
|
||||
orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION,
|
||||
orchestrationRequestId: 'update_1',
|
||||
method: 'orchestration.taskUpdate',
|
||||
params: {
|
||||
id: 'task_t',
|
||||
status: 'completed',
|
||||
result: 'by hand',
|
||||
callerTerminalHandle: 'term_coord',
|
||||
run: run.id
|
||||
}
|
||||
})
|
||||
expect(response).toMatchObject({
|
||||
ok: true,
|
||||
result: { on: 'devB', task: { status: 'completed' } }
|
||||
})
|
||||
const forwarded = sent.find((s) => s.method === 'orchestration.taskUpdate')
|
||||
expect(forwarded?.params).toEqual({ id: 'task_t', status: 'completed', result: 'by hand' })
|
||||
expect(forwarded?.envelope?.orchestrationRequestId).toMatch(/^fwd:/)
|
||||
expect(db.getTask('task_t')?.status).toBe('dispatched')
|
||||
})
|
||||
})
|
||||
|
||||
describe('S8 pull loop', () => {
|
||||
it('acknowledges what it imported before an unreadable item, and never that item', async () => {
|
||||
startedOnB()
|
||||
|
||||
@@ -561,6 +561,36 @@ describe('commands about a worker a paired server owns', () => {
|
||||
expect(getRemoteDispatch(db, 'ctx_b1')).toMatchObject({ stop_owed: 1 })
|
||||
})
|
||||
|
||||
it("updates this Orca's own Task here by the local rule, and never sends the update to its server", async () => {
|
||||
const { taskId, runId } = await startOnB()
|
||||
importState(runId, taskId, 1, {
|
||||
status: 'dispatched',
|
||||
workerState: 'ready',
|
||||
taskStatus: 'dispatched'
|
||||
})
|
||||
|
||||
const held = await call('orchestration.taskUpdate', {
|
||||
id: taskId,
|
||||
status: 'completed',
|
||||
result: 'by hand',
|
||||
callerTerminalHandle: 'term_coord'
|
||||
})
|
||||
expect(held.ok ? undefined : held.error.message).toContain(
|
||||
`Task ${taskId} cannot move to completed while supervised Dispatch ctx_b1 is active`
|
||||
)
|
||||
|
||||
await call('orchestration.workerStop', { dispatch: 'ctx_b1' })
|
||||
const updated = await call('orchestration.taskUpdate', {
|
||||
id: taskId,
|
||||
status: 'failed',
|
||||
result: 'given up',
|
||||
callerTerminalHandle: 'term_coord'
|
||||
})
|
||||
expect(updated.ok).toBe(true)
|
||||
expect(db.getTask(taskId)?.status).toBe('failed')
|
||||
expect(sent.map((s) => s.method)).not.toContain('orchestration.taskUpdate')
|
||||
})
|
||||
|
||||
/** devB's queue item saying where its Dispatch and Task now stand. */
|
||||
function importState(
|
||||
runId: string,
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
import { z } from 'zod'
|
||||
import type { TaskStatus } from '../../db/orchestration-records'
|
||||
import { currentRemoteAttempt, remoteHoldOnTask } from '../../db/remote-owned/remote-placements'
|
||||
import { getOpenRemoteRunLink } from '../../db/remote-owned/remote-run-links'
|
||||
import {
|
||||
callRemoteOwnedServer,
|
||||
remoteForwardRequestId,
|
||||
type RemoteOwnedRuntime
|
||||
} from './remote-owned-calls'
|
||||
|
||||
const TASK_UPDATE_TIMEOUT_MS = 30_000
|
||||
|
||||
/**
|
||||
* A Task a paired server still holds (its attempt runs, or its result has not come home) is updated there by its ordinary
|
||||
* taskUpdate, which owns the worker the update must respect. The Task here changes only when the
|
||||
* result comes back through the server's queue. Undefined when the Task is not held there.
|
||||
*/
|
||||
export async function forwardRemoteTaskUpdate(
|
||||
runtime: RemoteOwnedRuntime,
|
||||
update: { id: string; status: TaskStatus; result?: string },
|
||||
retry: { callerNamespace: string; requestId: string } | undefined
|
||||
): Promise<{ task: unknown; on: string } | undefined> {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const attempt = currentRemoteAttempt(db, update.id)
|
||||
const link = attempt && getOpenRemoteRunLink(db, attempt.conversation_id)
|
||||
if (!attempt || !link || remoteHoldOnTask(db, update.id)?.dispatchId !== attempt.dispatch_id) {
|
||||
return undefined
|
||||
}
|
||||
const answer = z.object({ task: z.unknown() }).safeParse(
|
||||
await callRemoteOwnedServer(runtime, link, {
|
||||
method: 'orchestration.taskUpdate',
|
||||
params: {
|
||||
id: update.id,
|
||||
status: update.status,
|
||||
...(update.result !== undefined ? { result: update.result } : {})
|
||||
},
|
||||
timeoutMs: TASK_UPDATE_TIMEOUT_MS,
|
||||
...(retry
|
||||
? { requestId: remoteForwardRequestId(retry.callerNamespace, retry.requestId) }
|
||||
: {}),
|
||||
whileLink: ['open']
|
||||
})
|
||||
)
|
||||
return {
|
||||
task: answer.success ? answer.data.task : db.getTask(update.id),
|
||||
on: runtime.resolveOrchestrationWorkerServer(link.environment_id).name
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,6 @@ import {
|
||||
TaskListParams
|
||||
} from '../../../../../../shared/rpc-contract/orchestration-params'
|
||||
import { TaskUpdateParams } from './task-update-params'
|
||||
import { forwardRemoteTaskUpdate } from '../../../../orchestration/federation/remote-owned/remote-owned-task-update'
|
||||
import { remoteForwardCallerNamespace } from '../../../../orchestration/federation/remote-owned/remote-forward-caller'
|
||||
|
||||
export const ORCHESTRATION_MESSAGE_METHODS = [
|
||||
ORCHESTRATION_REPLY_METHOD,
|
||||
@@ -127,26 +125,8 @@ export const ORCHESTRATION_MESSAGE_METHODS = [
|
||||
targets: { id: 'task', run: 'run' },
|
||||
rule: taskUpdateAccess
|
||||
},
|
||||
handler: async (params, ctx) => {
|
||||
const { orchestrationMutation, runtime } = ctx
|
||||
handler: (params, { runtime }) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const forwarded = await forwardRemoteTaskUpdate(
|
||||
runtime,
|
||||
params,
|
||||
orchestrationMutation
|
||||
? {
|
||||
callerNamespace: remoteForwardCallerNamespace(
|
||||
runtime,
|
||||
ctx,
|
||||
params.callerTerminalHandle
|
||||
),
|
||||
requestId: orchestrationMutation.requestId
|
||||
}
|
||||
: undefined
|
||||
)
|
||||
if (forwarded) {
|
||||
return forwarded
|
||||
}
|
||||
const task = db.updateTaskStatus(params.id, params.status, params.result)
|
||||
if (!task) {
|
||||
throw new Error(`Task not found: ${params.id}`)
|
||||
|
||||
Reference in New Issue
Block a user