diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 6d80f19197..8c1ca431b9 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -92,7 +92,7 @@ export interface BenchmarkWorkspaceResource { } export interface BenchmarkWorkspaceJob { - /** Stable id so a case prompt can reference a specific run (e.g. for get_job_logs). */ + /** Stable id so a case prompt can reference a specific run (e.g. for get_run). */ id?: string jobKind?: CompletedJob['job_kind'] scriptPath?: string @@ -100,6 +100,8 @@ export interface BenchmarkWorkspaceJob { label?: string success?: boolean logs?: string + args?: Record + result?: unknown } export interface BenchmarkWorkspaceRunnables { @@ -156,7 +158,7 @@ export function registerBenchmarkWorkspaceRunnables( ...runnables, datatables: runnables.datatables ? structuredClone(runnables.datatables) : undefined }) - // Seed any fixture jobs so list_runs / get_job_logs have data to return. + // Seed any fixture jobs so list_runs / get_run have data to return. for (const seed of runnables.jobs ?? []) { createBenchmarkCompletedJob({ workspace, @@ -166,7 +168,9 @@ export function registerBenchmarkWorkspaceRunnables( scriptPath: seed.scriptPath, createdBy: seed.createdBy, label: seed.label, - logs: seed.logs + logs: seed.logs, + args: seed.args, + result: seed.result }) } } @@ -481,6 +485,33 @@ export function getBenchmarkJobLogs(workspace: string, jobId: string): string { return job.logs ?? '' } +/** + * Mirror `JobService.getFlowAllResults`, which get_run calls for the execution + * tree. Fixture jobs are single runs with no steps, so only the root entry. + */ +export function getBenchmarkFlowAllResults(workspace: string, jobId: string) { + const job = getBenchmarkCompletedJob(workspace, jobId) + if (!job) { + throw new Error(`Job "${jobId}" not found in benchmark workspace`) + } + return { + entries: [ + { + job_id: jobId, + label: 'Flow', + kind: job.job_kind ?? 'script', + depth: 0, + sibling_index: 1, + sibling_count: 1, + status: job.success ? 'success' : 'failure', + success: job.success + } + ], + truncated: false, + scope_filtered: false + } +} + // ============= Drafts (per-user, DB-backed in production) ============= /** diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index e5b275b86e..f240d0ee36 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -62,6 +62,7 @@ vi.mock('$lib/gen', async () => { getBenchmarkDatatableSchema, getBenchmarkDraftForUser, getBenchmarkFlowByPath, + getBenchmarkFlowAllResults, getBenchmarkJobLogs, getBenchmarkOwnDraft, getBenchmarkScriptByHash, @@ -325,7 +326,11 @@ vi.mock('$lib/gen', async () => { getJobLogs: async (data: { workspace: string; id: string }) => hasBenchmarkWorkspace(data.workspace) ? getBenchmarkJobLogs(data.workspace, data.id) - : actual.JobService.getJobLogs(data) + : actual.JobService.getJobLogs(data), + getFlowAllResults: async (data: { workspace: string; id: string }) => + hasBenchmarkWorkspace(data.workspace) + ? getBenchmarkFlowAllResults(data.workspace, data.id) + : actual.JobService.getFlowAllResults(data) }), WorkspaceService: wrapService(actual.WorkspaceService, { getCopilotInfo: async (data: { workspace: string }) => diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 71693156a9..951d316524 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -889,13 +889,13 @@ draftCountExactly: 0 toolExpect: requiredToolsUsed: - - get_job_logs + - get_run forbiddenToolsUsed: - deploy_workspace_item - delete_workspace_item - write_script toolCallArgs: - - tool: get_job_logs + - tool: get_run field: id stringIncludesAnyOf: - 01920000-0000-7000-8000-0000000000f1 @@ -906,6 +906,34 @@ - fetches the logs for the requested job id - explains the failure from the returned logs (connection refused to the upstream API) +- id: global-run-args-and-result + prompt: |- + What was the run 01920000-0000-7000-8000-0000000000f2 called with, and what did it return? + initial: ai_evals/fixtures/frontend/global/initial/jobs_seed.json + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - get_run + forbiddenToolsUsed: + - test_run_script + - run_script + - deploy_workspace_item + toolCallArgs: + - tool: get_run + field: id + stringIncludesAnyOf: + - 01920000-0000-7000-8000-0000000000f2 + # Read-only, so no draft for the global judge to score ā€” validated on tool use + # and the deterministic argument check, like the neighbouring run cases. + skipJudge: true + judgeChecklist: + - reports the arguments the run was called with (region emea, 12 recipients) + - reports what the run returned (12 sent, 3 skipped) + - does not start a new run to find out + # --- Page navigation (open_page) --- # The assistant should take the user to a Windmill page (Runs/Schedules) with the # right filters via open_page, rather than describing where to click or dumping the diff --git a/ai_evals/fixtures/frontend/global/initial/jobs_seed.json b/ai_evals/fixtures/frontend/global/initial/jobs_seed.json index b075d6df26..4c0f923553 100644 --- a/ai_evals/fixtures/frontend/global/initial/jobs_seed.json +++ b/ai_evals/fixtures/frontend/global/initial/jobs_seed.json @@ -15,6 +15,8 @@ "jobKind": "script", "createdBy": "bob", "success": true, + "args": { "region": "emea", "dry_run": false, "recipients": 12 }, + "result": { "sent": 12, "skipped": 3, "digest_url": "https://reports.example.com/d/2026-06-09" }, "logs": "Generating daily digest...\nDigest emailed to 12 recipients\nDone in 1.2s" }, { diff --git a/frontend/src/lib/components/copilot/ScriptFix.svelte b/frontend/src/lib/components/copilot/ScriptFix.svelte index f586ad1aaf..78c4804cc8 100644 --- a/frontend/src/lib/components/copilot/ScriptFix.svelte +++ b/frontend/src/lib/components/copilot/ScriptFix.svelte @@ -25,7 +25,7 @@ /** The failing run's error, used when there is no job to point at. */ error?: string /** The failing run's job id. Preferred over `error`: the chat reads the - * run itself with `get_job_logs`, which gives it the logs rather than + * run itself with `get_run`, which gives it the logs rather than * just the thrown value, and keeps the composer readable. */ jobId?: string /** Set when this sits in a flow step's preview, so the session opens on diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts index 0b1ea0952e..a78f09071a 100644 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts @@ -31,6 +31,18 @@ const CATALOG = [ properties: { page: { type: 'integer' }, per_page: { type: 'integer' } } } }, + { + name: 'getJobUpdates', + description: 'Get job updates', + instructions: '', + path: '/w/{workspace}/jobs_u/getupdate/{id}', + method: 'GET', + path_params_schema: { + type: 'object', + properties: { workspace: { type: 'string' }, id: { type: 'string' } }, + required: ['workspace', 'id'] + } + }, { name: 'getJob', description: 'Get job details', @@ -195,6 +207,9 @@ describe('call_api_get', () => { const mutating = await run('call_api_get', { name: 'cancelQueuedJob' }) expect(mutating.error).toContain('call_api_endpoint') + const job = await run('call_api_get', { name: 'getJob' }) + expect(job.error).toContain('get_run') + const deleting = await run('call_api_endpoint', { name: 'deleteSchedule' }) expect(deleting.error).toContain('delete_workspace_item') @@ -248,7 +263,7 @@ describe('call_api_get', () => { }) it('returns the endpoint schema when a required path param is missing', async () => { - const result = await run('call_api_get', { name: 'getJob' }) + const result = await run('call_api_get', { name: 'getJobUpdates' }) expect(result.success).toBe(false) expect(result.error).toContain('id') expect(result.schema.path_params_schema.required).toContain('id') diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts index 42d45916eb..308d222544 100644 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts +++ b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts @@ -58,7 +58,8 @@ const COVERED_ENDPOINTS: Record = { searchDocs: 'search_docs', readDocsPage: 'read_docs_page', listJobs: 'list_runs', - getJobLogs: 'get_job_logs', + getJob: 'get_run', + getJobLogs: 'get_run', runScriptPreviewAndWaitResult: 'test_run_script' } diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 051304cafa..fe0477a544 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -85,10 +85,33 @@ vi.mock('$lib/gen', async () => { runScriptByPath: vi.fn(async () => 'job-script-by-path'), getJob: vi.fn(async () => ({ type: 'CompletedJob', + id: 'job-123', + job_kind: 'script', + script_path: 'f/team/runner', success: true, + canceled: false, + args: { n: 3 }, result: { ok: true }, logs: 'test logs' })), + getJobArgs: vi.fn(async () => ({ big: 'real args' })), + getCompletedJobResultMaybe: vi.fn(async () => ({ completed: true, result: { big: 'x' } })), + getFlowAllResults: vi.fn(async () => ({ + entries: [ + { + job_id: 'job-123', + label: 'Flow', + kind: 'script', + depth: 0, + sibling_index: 1, + sibling_count: 1, + status: 'success', + success: true + } + ], + truncated: false, + scope_filtered: false + })), // What every job wait polls first; unmocked it reaches the real client and the // wait never returns. Answers completed, so one tick settles the job. getJobUpdates: vi.fn(async () => ({ @@ -489,7 +512,7 @@ describe('global AI tools', () => { expect(names).toContain('test_run_script') expect(names).toContain('test_run_flow') expect(names).toContain('test_run_step') - expect(names).toContain('get_job_logs') + expect(names).toContain('get_run') expect(names).toContain('list_runs') }) @@ -715,29 +738,210 @@ describe('global AI tools', () => { ) }) - it('fetches job logs by id and always suppresses the backend ansi hint line', async () => { - const result = await callGlobalTool('get_job_logs', { id: 'job-123' }) + it('returns args, result and logs of a run in one call', async () => { + const result = await callGlobalTool('get_run', { id: 'job-123' }) expect(JobService.getJobLogs).toHaveBeenCalledWith({ workspace: WORKSPACE, id: 'job-123', + // The backend's "to remove ansi colors, use: sed ..." hint is noise for + // the model, so it is always suppressed. removeAnsiWarnings: true }) - expect(result).toBe('job log line 1\njob log line 2') - // The logs must be surfaced as the tool result so the details panel shows - // them rather than "No result yet". + const parsed = JSON.parse(result) + expect(parsed.run).toMatchObject({ + status: 'success', + path: 'f/team/runner', + args: expect.stringContaining('"n": 3'), + result: expect.stringContaining('"ok": true'), + logs: 'job log line 1\njob log line 2' + }) + // The result must be surfaced as the tool result so the details panel shows + // it rather than "No result yet". expect(toolCallbacks.setToolStatus).toHaveBeenCalledWith( - 'test-get_job_logs', - expect.objectContaining({ result: 'job log line 1\njob log line 2' }) + 'test-get_run', + expect.objectContaining({ result }) ) }) - it('reports when a job has no logs', async () => { + it('reports when a run has no logs, and tells that apart from logs it could not read', async () => { vi.mocked(JobService.getJobLogs).mockResolvedValueOnce(' ') + expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-empty' })).run.logs).toBe( + 'No logs for this run.' + ) - const result = await callGlobalTool('get_job_logs', { id: 'job-empty' }) + // A failed fetch must not read as "this run logged nothing" ā€” the model + // would report that to the user as fact. + vi.mocked(JobService.getJobLogs).mockRejectedValueOnce(new Error('boom')) + expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })).run.logs).toBe( + 'Logs could not be read for this run.' + ) - expect(result).toBe('No logs available for this job.') + // Nor does every failed read reject: the generated client resolves undefined + // when it cannot read the body, which lands on the same "no logs" branch. + vi.mocked(JobService.getJobLogs).mockResolvedValueOnce(undefined as any) + expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })).run.logs).toBe( + 'Logs could not be read for this run.' + ) + }) + + it('keeps the end of a long log, and never opens it on half a surrogate pair', async () => { + // 12002 code points over 24003 UTF-16 units: the 12000-unit tail opens one + // unit into a unicorn, so the lone low surrogate has to be dropped. + vi.mocked(JobService.getJobLogs).mockResolvedValueOnce('šŸ¦„'.repeat(12001) + 'z') + + const [note, body] = JSON.parse( + await callGlobalTool('get_run', { id: 'job-123' }) + ).run.logs.split('\n') + + // The note goes first: the tail is what the model came for, and a note at + // the end would read as the last thing the run logged. + expect(note).toContain('12002 chars total') + expect(body).toHaveLength(11999) + expect(body.codePointAt(0)).toBe(0x1f984) + expect(body.endsWith('šŸ¦„z')).toBe(true) + }) + + it('keeps the step tree optional: a failed tree fetch still returns the run itself', async () => { + vi.mocked(JobService.getFlowAllResults).mockRejectedValueOnce(new Error('tree unavailable')) + + const parsed = JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })) + + expect(parsed.run).toMatchObject({ args: expect.stringContaining('"n": 3') }) + expect(parsed.run.logs).toBe('job log line 1\njob log line 2') + // Silence here would read as "this flow ran no steps", which the model + // would then report to the user as fact. + expect(parsed.run.steps_unavailable).toBe(true) + // The tree's root entry normally names the run; without it nothing does. + expect(parsed.run.job_id).toBe('job-123') + + // And the tree read fails the same two ways the log read does: reading + // `.entries` off a resolved undefined throws, which would cost the model the + // job and logs already in hand rather than just the tree. + vi.mocked(JobService.getFlowAllResults).mockResolvedValueOnce(undefined as any) + const noTree = JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })) + expect(noTree.run.steps_unavailable).toBe(true) + expect(noTree.run.logs).toBe('job log line 1\njob log line 2') + }) + + it('reports why a run was canceled or died, the fields getJob used to carry', async () => { + vi.mocked(JobService.getJob).mockResolvedValueOnce({ + type: 'CompletedJob', + id: 'job-oom', + job_kind: 'script', + success: false, + canceled: true, + canceled_by: 'alice', + canceled_reason: 'exceeded memory limit', + mem_peak: 2097152 + } as any) + + const run = JSON.parse(await callGlobalTool('get_run', { id: 'job-oom' })).run + + expect(run.status).toBe('canceled') + expect(run.canceled_by).toBe('alice') + expect(run.canceled_reason).toBe('exceeded memory limit') + expect(run.mem_peak_kb).toBe(2097152) + }) + + // Over ~90KB the job endpoint elides the payload, leaving the step tree as the + // only place a real (server-side truncated) head of the result survives. + function mockElidedJob() { + vi.mocked(JobService.getJob).mockResolvedValueOnce({ + type: 'CompletedJob', + id: 'job-big', + job_kind: 'script', + success: true, + canceled: false, + args: { reason: 'WINDMILL_TOO_BIG' }, + result: 'WINDMILL_TOO_BIG' + } as any) + vi.mocked(JobService.getFlowAllResults).mockResolvedValueOnce({ + entries: [ + { + job_id: 'job-big', + label: 'Flow', + kind: 'script', + depth: 0, + sibling_index: 1, + sibling_count: 1, + status: 'success', + success: true, + result_prefix: '{"blob":"real head"}', + result_length: 300018 + } + ], + truncated: false, + scope_filtered: false + } as any) + } + + it("reports getJob's WINDMILL_TOO_BIG placeholders instead of fetching around them", async () => { + mockElidedJob() + + const run = JSON.parse(await callGlobalTool('get_run', { id: 'job-big' })).run + + // The marker is never the run's own value, so it must not reach the model. + expect(JSON.stringify(run)).not.toContain('WINDMILL_TOO_BIG') + // The result falls to the step tree's real server-side head, flagged and + // sized so the model can't mistake the fragment for the whole payload. + expect(run.result).toBe('{"blob":"real head"}') + expect(run.result_total_chars).toBe(300018) + expect(run.result_truncated).toBe(true) + expect(run.args_truncated).toBe(true) + expect(run.args).toBeUndefined() + // The endpoints that return these whole take no length parameter, so + // reaching for one would pull the entire payload into the tab. + expect(JobService.getJobArgs).not.toHaveBeenCalled() + expect(JobService.getCompletedJobResultMaybe).not.toHaveBeenCalled() + }) + + it('keeps a payload that merely carries the marker string in a reason of its own', async () => { + // The backend elides a result to the bare string and args to exactly + // {reason: marker}. A payload with that reason plus fields of its own is the + // run's own value, and withholding it would report an elision that never was. + vi.mocked(JobService.getJob).mockResolvedValueOnce({ + type: 'CompletedJob', + id: 'job-reason', + job_kind: 'script', + success: false, + canceled: false, + args: { reason: 'WINDMILL_TOO_BIG', retries: 2 }, + result: { reason: 'WINDMILL_TOO_BIG', code: 42 } + } as any) + + const run = JSON.parse(await callGlobalTool('get_run', { id: 'job-reason' })).run + + expect(run.args_truncated).toBeUndefined() + expect(run.result_truncated).toBeUndefined() + expect(run.args).toContain('"retries": 2') + expect(run.result).toContain('"code": 42') + }) + + it('reports skipped and suspended runs as such rather than success or running', async () => { + // `success` is true for a skipped job, and a suspended job is `running`. + vi.mocked(JobService.getJob).mockResolvedValueOnce({ + type: 'CompletedJob', + id: 'job-skipped', + job_kind: 'script', + success: true, + canceled: false, + is_skipped: true + } as any) + expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-skipped' })).run.status).toBe( + 'skipped' + ) + + vi.mocked(JobService.getJob).mockResolvedValueOnce({ + type: 'QueuedJob', + id: 'job-suspended', + job_kind: 'flow', + running: true, + suspend: 1 + } as any) + expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-suspended' })).run.status).toBe( + 'suspended' + ) }) it('searches hub scripts without fetching script contents', async () => { @@ -6302,7 +6506,7 @@ describe('prepareGlobalSystemMessage', () => { it('dispatches to the registered handler with the session id and default limit of 20', async () => { const callbacks: ToolCallbacks = { setToolStatus: vi.fn(), removeToolStatus: vi.fn() } const handler = vi.fn(() => ({ - aiResult: 'runs output. Next step: call get_job_logs.', + aiResult: 'runs output. Next step: call get_run.', uiMessage: 'Listed 1 app run', toolResult: '[{"job_id":"job-1","component":"backend.1","status":"completed","created_at":1718000000000,"started_at":1718000000000,"duration_ms":1000}]' @@ -6311,7 +6515,7 @@ describe('prepareGlobalSystemMessage', () => { const result = await callGlobalTool('list_app_runs', {}, callbacks, { sessionId: 'sess-runs' }) - expect(result).toBe('runs output. Next step: call get_job_logs.') + expect(result).toBe('runs output. Next step: call get_run.') expect(handler).toHaveBeenCalledWith({ sessionId: 'sess-runs', limit: 20 }) expect(callbacks.setToolStatus).toHaveBeenLastCalledWith('test-list_app_runs', { content: 'Listed 1 app run', diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 52f4ee39ef..1511bf24a6 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -30,7 +30,6 @@ import type { Flow, FlowModule, FlowValue, - Job, ListableApp, ListableResource, ListableVariable, @@ -187,7 +186,7 @@ import { getDraftDiffValues } from '$lib/utils_draft_deploy' import { changedLineIndices, draftDeployedPatch, windowPatch } from './draftDiff' -import { getFlowRunDetails } from './flowRunTree' +import { getRun, summarizeRun } from './flowRunTree' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { invalidateWorkspaceComparison } from '$lib/workspaceComparison' import type { UserDraftItemKind } from '$lib/gen' @@ -693,17 +692,13 @@ const searchResourceTypesSchema = z.object({ .describe('Max number of resource types to return. Defaults to 5.') }) -const getJobLogsSchema = z.object({ - id: z.string().describe('The UUID of the job to fetch logs for.') -}) - -const getFlowRunDetailsSchema = z.object({ - id: z.string().describe('The UUID of the flow run to inspect.'), +const getRunSchema = z.object({ + id: z.string().describe('The UUID of the run (job) to inspect.'), step: z .string() .optional() .describe( - 'Step to drill into for its result (returned in full up to 12k chars), addressed by the step ids shown in the tree: "b" for a top-level step, "b/c" for a step inside a subflow, "b[12]" for iteration 12 of a loop or attempt 12 of a retried step (1-based), composable as "b[12]/c". Omit to get the whole per-step tree.' + 'Step to drill into for its result (returned in full up to 12k chars), addressed by the step ids shown in the tree: "b" for a top-level step, "b/c" for a step inside a subflow, "b[12]" for iteration 12 of a loop or attempt 12 of a retried step (1-based), composable as "b[12]/c". Omit to get the run itself.' ) }) @@ -1356,8 +1351,8 @@ Rules: ${pipelineBullet} - After creating or editing a script or flow draft, run test_run_script, test_run_flow, or test_run_step with representative args before reporting that it works. These tools prefer drafts, so testing does not require deployment. - Do the same for a raw app: run test_run_app_runnable on each backend runnable you wrote or changed before saying the app works. A bundle that compiles proves nothing about whether the runnables run. An inline runnable executes the app's draft code; a path runnable executes the DEPLOYED script/flow it names, so a path runnable aimed at something you have not deployed fails here ā€” that failure is the point: report it and offer to deploy that one target. The app itself does not need deploying to be tested. -- Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_job_logs with a returned id to inspect a specific run's logs ā€” without starting a new test run. -- To see what a flow run actually did per step ā€” statuses and results across the whole execution tree, subflow steps and loop iterations included ā€” use get_flow_run_details with the run id (it also works while the flow is still running). Pass step to read one step's result in full (capped at 12k chars). Prefer it over get_job_logs when you need step results rather than logs. +- Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_run with a returned id to see what that run was called with, what it returned and what it logged ā€” without starting a new test run. +- get_run also covers what a flow run did per step ā€” statuses and results across the whole execution tree, subflow steps and loop iterations included ā€” and works while the flow is still running. Pass step to read one step's result in full (capped at 12k chars). - Use open_page to show a workspace page with filters applied ā€” Runs, Schedules, Variables, Resources, Assets, Audit logs, or Workspace settings on a specific tab (e.g. "open the failed runs of f/foo/bar", "open the schedule for X", "open the git sync settings"). Carry over every filter the user described ā€” Runs takes the page's whole filter set (time window, path, user, folder, label, tag, worker, trigger kind, args/result, ...), so don't drop a criterion just because it wasn't in the request's main clause. Only the pages listed for this user in the tool are available; don't offer pages that aren't listed. Don't use it as a substitute for list_runs when you just need the data yourself. - Whenever you ask the user to perform a manual step in the UI ā€” fill in a resource's credentials, set a secret variable's value, adjust a schedule or setting ā€” call open_page in the same message, targeted at that item (pass open with its path to land in its edit drawer, or the page's filters otherwise). Never just describe where to click. - When the user is happy with the changes and wants to review or deploy them, use open_page with page "compare" ā€” it opens the Compare & Deploy review page.${ @@ -1376,7 +1371,7 @@ ${pipelineBullet} - Building a data pipeline: call open_preview(kind="pipeline", path="") as the FIRST step, before creating any node ā€” this opens the pipeline editor the user reviews in. path is the folder, not an item; an empty or not-yet-created folder is fine (create_folder first if needed, then open it). Opening it registers build_pipeline_node / edit_pipeline_node ā€” use ONLY those to add or change pipeline nodes, never write_script for a pipeline node ā€” they apply directly as unsaved drafts on the canvas (no separate accept/reject step) that the user reviews and deploys. Do not write pipeline scripts without first opening the editor. - When debugging a running raw app, call get_app_runtime_logs to read the live preview's browser console output. It needs the raw app preview open (open_preview kind="raw_app"). - To inspect what actually rendered in a running raw app (verify an edit landed on screen, diagnose a blank/empty or wrong view, answer "what's showing"), use search_dom (regex over the live HTML) and read_dom (a line-numbered window). Pass a \`selector\` to scope to an element ā€” prefer the selector from a DOM element chip the user attached ā€” or omit it for the whole page. When a chip lists an \`app_path\`, pass it too so the RIGHT app is read (several previews can be open; a query without \`app_path\` hits the visible one). The DOM is read live and is never in context; no match means the element isn't rendered. Both need the raw app preview open. -- get_app_runtime_logs only shows the app's browser console. For the server-side logs of a backend runnable the app invoked (a backend. call), call list_app_runs to get that run's job_id from the live preview, then get_job_logs with it. Use this when a backend call errors or returns something unexpected. +- get_app_runtime_logs only shows the app's browser console. For the server-side logs of a backend runnable the app invoked (a backend. call), call list_app_runs to get that run's job_id from the live preview, then get_run with it. Use this when a backend call errors or returns something unexpected. ${ isChromiumBrowser() ? `- When the user raises how a raw app looks (something is off, or they want the design or layout improved), call take_screenshot to see what they are looking at before changing anything. Reach for it when the request is about appearance, not to review your own edits, which you can read back from the code. It needs the raw app preview open (open_preview kind="raw_app").` @@ -1555,36 +1550,6 @@ function variableToItem(variable: ListableVariable): WorkspaceItem { } } -// Compact metadata for one run. The raw Job carries args/result/logs/raw_code -// which can be huge ā€” list_runs returns only what's needed to identify a run. -function summarizeRun(job: Job): Record { - const base = { - id: job.id, - job_kind: job.job_kind, - path: job.script_path, - created_by: job.created_by, - created_at: job.created_at, - started_at: job.started_at, - schedule_path: job.schedule_path, - is_flow_step: job.is_flow_step, - tag: job.tag, - worker: job.worker - } - if ('success' in job) { - // CompletedJob - return { - ...base, - status: job.canceled ? 'canceled' : job.success ? 'success' : 'failure', - duration_ms: job.duration_ms - } - } - // QueuedJob (running or still waiting in the queue) - return { - ...base, - status: job.running ? 'running' : 'queued' - } -} - // ============= App helpers ============= type BackendRunnableInput = z.infer @@ -3752,7 +3717,7 @@ export const globalTools: Tool<{}>[] = [ def: createToolDef( listRunsSchema, 'list_runs', - "List recent runs (jobs), most recent first. Optionally filter by path, creator, label, or status. Returns compact metadata only ā€” use get_job_logs with a returned id to read a run's logs." + 'List recent runs (jobs), most recent first. Optionally filter by path, creator, label, or status. Returns compact metadata only ā€” use get_run with a returned id to see what a run was called with, returned and logged.' ), planModeSafe: true, showDetails: true, @@ -3779,56 +3744,24 @@ export const globalTools: Tool<{}>[] = [ }, { def: createToolDef( - getFlowRunDetailsSchema, - 'get_flow_run_details', - "Inspect a flow run's execution tree: per-step statuses and truncated results, including subflow steps, loop iterations, branches, and retries. Works on running flows too. Pass step to fetch one step's result in full (up to 12k chars)." + getRunSchema, + 'get_run', + "Inspect one run: its status, arguments, result and logs, plus ā€” for a flow ā€” the per-step execution tree with each step's status and truncated result (subflow steps, loop iterations, branches and retries included). Works on running jobs too. Pass step to fetch one step's result in full (up to 12k chars)." ), planModeSafe: true, showDetails: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = getFlowRunDetailsSchema.parse(args) + const parsed = getRunSchema.parse(args) toolCallbacks.setToolStatus(toolId, { content: parsed.step ? `Fetching result of step ${parsed.step} in run ${parsed.id}...` - : `Inspecting flow run ${parsed.id}...` + : `Inspecting run ${parsed.id}...` }) - const result = await getFlowRunDetails(workspace, parsed.id, parsed.step) + const result = await getRun(workspace, parsed.id, parsed.step) toolCallbacks.setToolStatus(toolId, { content: parsed.step ? `Fetched result of step ${parsed.step} in run ${parsed.id}` - : `Inspected flow run ${parsed.id}`, - result - }) - return result - } - }, - { - def: createToolDef( - getJobLogsSchema, - 'get_job_logs', - 'Fetch the logs of a job by its id. Use this to inspect the output of an existing run.' - ), - planModeSafe: true, - showDetails: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = getJobLogsSchema.parse(args) - toolCallbacks.setToolStatus(toolId, { - content: `Fetching logs for job ${parsed.id}...` - }) - const logs = await JobService.getJobLogs({ - workspace, - id: parsed.id, - // Always suppress the "to remove ansi colors, use: sed ..." hint the - // backend otherwise prepends ā€” it is noise for the model and is not - // actual ANSI stripping (the raw logs are returned either way). - removeAnsiWarnings: true - }) - const hasLogs = typeof logs === 'string' && logs.trim().length > 0 - const result = hasLogs ? logs : 'No logs available for this job.' - toolCallbacks.setToolStatus(toolId, { - content: hasLogs - ? `Fetched logs for job ${parsed.id}` - : `No logs available for job ${parsed.id}`, + : `Inspected run ${parsed.id}`, result }) return result @@ -6569,7 +6502,7 @@ async function testRunAppRunnable( toolId, startMessage: `Running backend runnable "${key}" of app "${path}"...`, // A path runnable pointing at a flow really does queue a flow job, so the - // failure path can offer get_flow_run_details; everything else is a script job. + // failure path can offer get_run's step tree; everything else is a script job. contextName: runnable.runType === 'flow' ? 'flow' : 'script', completionName: 'backend runnable', background: args.background, diff --git a/frontend/src/lib/components/copilot/chat/global/flowRunTree.test.ts b/frontend/src/lib/components/copilot/chat/global/flowRunTree.test.ts index 4b0dee5de0..7b2ddeb5fc 100644 --- a/frontend/src/lib/components/copilot/chat/global/flowRunTree.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/flowRunTree.test.ts @@ -190,4 +190,29 @@ describe('shapeFlowRunTree', () => { expect(parsed.steps[1].result).toBeUndefined() expect(parsed.steps[1].result_total_chars).toBe(700) }) + + it('spends the tree budget on the tree, not on the run payloads carried with it', () => { + const entries = [root()] + for (let i = 1; i <= 12; i++) { + entries.push( + entry({ + step_path: `s${i}`, + flow_step_id: `s${i}`, + status: 'failure', + success: false, + result_prefix: 'y'.repeat(700), + result_length: 700 + }) + ) + } + const logs = 'L'.repeat(12000) + const rendered = shapeFlowRunTree({ entries }, { status: 'failure', logs }) + + // Counting the overrides against the budget would shrink this tree away + // even though the tree itself fits: the model asked for the payloads. + expect(rendered.length).toBeGreaterThan(20000) + const parsed = JSON.parse(rendered) + expect(parsed.run.logs).toBe(logs) + expect(parsed.steps[0].result.length).toBe(700) + }) }) diff --git a/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts b/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts index 207dca1789..7f63de373b 100644 --- a/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts +++ b/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts @@ -1,19 +1,25 @@ -import { JobService, type GetFlowAllResultsResponse } from '$lib/gen' +import { JobService, type Job, type GetFlowAllResultsResponse } from '$lib/gen' +import { isWindmillTooBigObject } from '$lib/components/job_args' /** - * Model-facing view of a flow run's execution tree for the global chat's - * get_flow_run_details tool. The backend endpoint (get_flow_all_results) - * enumerates every job of the tree with per-entry truncated results; this - * module shapes that flat list into a compact per-step tree the model can - * read in one tool result. Step addresses ('b/c', 'b[12]/c') are resolved - * server-side by the same endpoint for full-result drill-down. + * Model-facing view of a run for the global chat's get_run tool: the job's own + * summary, args, result and logs, plus ā€” when the run has steps ā€” its execution + * tree. The backend endpoint (get_flow_all_results) enumerates every job of the + * tree with per-entry truncated results; this module shapes that flat list into + * a compact per-step tree the model can read in one tool result. Step addresses + * ('b/c', 'b[12]/c') are resolved server-side by the same endpoint for + * full-result drill-down. + * + * `summarizeRun` also backs list_runs' per-job summary, so it lives here rather + * than in core.ts: get_run needs it, and importing it back would be circular. */ export type FlowResultEntry = GetFlowAllResultsResponse['entries'][number] /** Per-entry result budget requested from the server for the tree view. */ export const TREE_RESULT_HEAD_CHARS = 700 -/** Cap on a drilled single-step full result handed to the model. */ +/** Cap on a full payload handed to the model: a drilled step result, and the + * run's own args, result and logs. */ export const STEP_RESULT_MAX_CHARS = 12000 /** Cap on the whole rendered tree; heads shrink progressively to fit. */ const TREE_TOTAL_BUDGET_CHARS = 20000 @@ -97,6 +103,13 @@ function sliceCodePointSafe(s: string, maxUnits: number): string { return last >= 0xd800 && last <= 0xdbff ? cut.slice(0, -1) : cut } +function sliceCodePointSafeEnd(s: string, maxUnits: number): string { + const cut = s.slice(-maxUnits) + const first = cut.charCodeAt(0) + // drop a leading lone low surrogate + return first >= 0xdc00 && first <= 0xdfff ? cut.slice(1) : cut +} + function shapeResult( entry: FlowResultEntry, opts: ShapeOpts @@ -211,7 +224,8 @@ function shapeChildren(children: FlowTreeNode[], opts: ShapeOpts): Record ): Record { const run = shapeStep(root, opts) const steps = run.steps @@ -221,19 +235,41 @@ function renderTree( if (!steps && root.entry.kind !== 'flow' && root.entry.kind !== 'flowpreview') { run.label = `Job (${root.entry.kind})` } + if (runOverrides) { + // A replacement result supersedes the tree entry's head, and the stale + // truncation marker must go with it. Without one the head stays: it is a + // real server-side prefix of a payload `getJob` would only hand back as a + // WINDMILL_TOO_BIG placeholder. + if ('result' in runOverrides) { + delete run.result + delete run.result_total_chars + } + Object.assign(run, runOverrides) + } return { ...(rootJobNote ? { note: rootJobNote } : {}), run, - ...(steps ? { steps } : {}), - hint: `Results are truncated. Call get_flow_run_details again with step="" (e.g. "b/c", or "b[12]" for one loop iteration) for a step's result in full (up to ${STEP_RESULT_MAX_CHARS} chars).` + ...(steps + ? { + steps, + hint: `Step results are truncated. Call get_run again with step="" (e.g. "b/c", or "b[12]" for one loop iteration) for a step's result in full (up to ${STEP_RESULT_MAX_CHARS} chars), or with id set to a step's job_id for that step's own args, result and logs.` + } + : {}) } } /** Render the whole tree, shrinking result heads until it fits the budget. */ -export function shapeFlowRunTree(response: GetFlowAllResultsResponse): string { +export function shapeFlowRunTree( + response: GetFlowAllResultsResponse, + runOverrides?: Record +): string { const root = buildFlowTree(response.entries) if (!root) { - return 'No jobs found for this run.' + // The tree is the optional half: what the caller already read of the job + // still answers the question, so don't drop it with the missing tree. + return runOverrides + ? JSON.stringify({ run: runOverrides }, null, 1) + : 'No jobs found for this run.' } const notes = [ ...(response.enclosing_job @@ -252,34 +288,188 @@ export function shapeFlowRunTree(response: GetFlowAllResultsResponse): string { ] const rootJobNote = notes.length > 0 ? notes.join(' ') : undefined + // The budget caps the tree, not the run's own args/result/logs ā€” those are + // capped on their own and the model asked for them, so they don't count here + // and the last-resort slice keeps room for them (`run` precedes `steps`). + const overrideChars = runOverrides ? JSON.stringify(runOverrides, null, 1).length : 0 let rendered = '' for (const opts of SHRINK_LADDER) { - rendered = JSON.stringify(renderTree(root, rootJobNote, opts), null, 1) - if (rendered.length <= TREE_TOTAL_BUDGET_CHARS) { + rendered = JSON.stringify(renderTree(root, rootJobNote, opts, runOverrides), null, 1) + if (rendered.length - overrideChars <= TREE_TOTAL_BUDGET_CHARS) { return rendered } } return ( - rendered.slice(0, TREE_TOTAL_BUDGET_CHARS) + + rendered.slice(0, TREE_TOTAL_BUDGET_CHARS + overrideChars) + `\nā€¦ (tree truncated at ${TREE_TOTAL_BUDGET_CHARS} chars ā€” drill into specific steps with the step parameter)` ) } -/** Entry point of the get_flow_run_details tool. Without `step`: the compact - * tree. With `step`: that job's full (capped) result, resolved server-side. */ -export async function getFlowRunDetails( - workspace: string, - id: string, - step?: string -): Promise { +// Compact metadata for one run. The raw Job carries args/result/logs/raw_code +// which can be huge ā€” this is only what's needed to identify a run, so list_runs +// can return one entry per job. +export function summarizeRun(job: Job): Record { + const base = { + id: job.id, + job_kind: job.job_kind, + path: job.script_path, + created_by: job.created_by, + created_at: job.created_at, + started_at: job.started_at, + schedule_path: job.schedule_path, + is_flow_step: job.is_flow_step, + tag: job.tag, + worker: job.worker + } + if ('success' in job) { + // CompletedJob. `success` is true for a skipped job too, so is_skipped has + // to be read first or a skipped step reports as a successful one. + return { + ...base, + status: job.canceled + ? 'canceled' + : job.is_skipped + ? 'skipped' + : job.success + ? 'success' + : 'failure', + duration_ms: job.duration_ms + } + } + // QueuedJob. A running job with suspends outstanding is parked ā€” on an approval + // step or on a parallelism slot ā€” not working, and `running` alone hides that. + return { + ...base, + status: job.running ? (job.suspend ? 'suspended' : 'running') : 'queued' + } +} + +/** Cap a payload to STEP_RESULT_MAX_CHARS. `tail` keeps the end instead of the + * start ā€” for logs, where the failure is at the bottom. + * + * Counts and cuts without materialising the string: logs arrive whole and + * unbounded, and spreading one into an array of code points costs ~9x its size + * (a 10MB log measured +90MB). Cutting in UTF-16 units keeps at most the budget + * in code points, never more, so an astral-heavy payload is trimmed slightly + * shorter than advertised rather than overshooting. */ +function cap(text: string, tail = false): string { + // UTF-16 length is never below the code-point count, so anything passing this + // is already under the cap and needs no counting pass at all. + if (text.length <= STEP_RESULT_MAX_CHARS) return text + const total = countCodePoints(text) + if (total <= STEP_RESULT_MAX_CHARS) return text + const note = `ā€¦ (truncated: ${total} chars total)` + return tail + ? `${note}\n${sliceCodePointSafeEnd(text, STEP_RESULT_MAX_CHARS)}` + : `${sliceCodePointSafe(text, STEP_RESULT_MAX_CHARS)}\n${note}` +} + +/** Told apart from an empty log so the model doesn't report "no logs" for a run + * whose logs it simply failed to read. */ +const LOGS_UNREADABLE = 'Logs could not be read for this run.' + +/** `getJob` swaps a payload over ~90KB for a marker rather than sending it + * (`get_job_query!` in backend/windmill-api/src/jobs.rs), and the two fields use + * different ones: a result becomes the bare string, args become exactly + * `{reason: }`. Each field matches only its own form, so a payload that + * merely carries that string in a `reason` of its own stays the run's value. */ +const TOO_BIG_RESULT = 'WINDMILL_TOO_BIG' + +function stringify(value: unknown): string { + return typeof value === 'string' ? value : JSON.stringify(value, null, 1) +} + +/** An elided args/result payload is reported, never fetched around: the + * endpoints that return those whole take no length parameter, so recovering a + * usable head would mean pulling the entire payload (up to MAX_RESULT_SIZE_MB, + * 500 by default) into the tab to keep 12k of it. The flag is the whole report ā€” + * the result keeps the head the step tree already carries beside it. */ +function shapeRunArgs(job: Job): Record { + if (!job.args) return {} + return isWindmillTooBigObject(job.args) + ? { args_truncated: true } + : { args: cap(stringify(job.args)) } +} + +function shapeRunResult(job: Job): Record { + if (!('result' in job) || job.result === undefined) return {} + // No `result` key when elided, so the tree's `result_prefix` ā€” a real + // server-side head of the same payload ā€” stands in its place unoverridden. + return job.result === TOO_BIG_RESULT + ? { result_truncated: true } + : { result: cap(stringify(job.result)) } +} + +/** Why a run ended badly. get_run is the only route to these ā€” the catalog + * refuses getJob. Kept out of summarizeRun, which list_runs pays per job. */ +function diagnoseRun(job: Job): Record { + return { + ...(job.canceled_by ? { canceled_by: job.canceled_by } : {}), + ...(job.canceled_reason ? { canceled_reason: job.canceled_reason } : {}), + ...(job.mem_peak ? { mem_peak_kb: job.mem_peak } : {}) + } +} + +/** Entry point of the get_run tool. Without `step`: the run's summary, args, + * result and logs, plus the per-step tree when the run has steps. With `step`: + * that step's full (capped) result, resolved server-side. */ +export async function getRun(workspace: string, id: string, step?: string): Promise { if (!step) { - return shapeFlowRunTree( - await JobService.getFlowAllResults({ workspace, id, maxResultLen: TREE_RESULT_HEAD_CHARS }) - ) + // Only the job read is load-bearing: logs and the step tree each answer + // part of the question, so neither failing should cost the model the rest. + const [job, logs, results] = await Promise.all([ + JobService.getJob({ workspace, id, noLogs: true, noCode: true }), + // The dedicated endpoint rather than the job's own `logs` field: that one + // is the last 20k still in the DB column, missing the head that log + // compaction flushed to object storage. This one stitches them back. + // + // It takes no length parameter, so unlike args and result the whole log + // does come into the tab before being capped. Only this job's own logs, + // though: a flow's are its orchestration lines, not its steps'. + JobService.getJobLogs({ + workspace, + id, + // Suppress the "to remove ansi colors, use: sed ..." hint the backend + // otherwise prepends ā€” noise for the model, and not actual stripping. + removeAnsiWarnings: true + }).catch(() => LOGS_UNREADABLE), + // `.catch` alone would leave the tree half-guarded: like the log read, this + // one can fail by resolving `undefined` rather than rejecting, and reading + // `.entries` off that throws ā€” losing the job and logs already in hand. + JobService.getFlowAllResults({ workspace, id, maxResultLen: TREE_RESULT_HEAD_CHARS }) + .catch(() => undefined) + .then((r) => r ?? ({ entries: [] } as GetFlowAllResultsResponse)) + ]) + const { id: _id, ...summary } = summarizeRun(job) + const payloads = { ...shapeRunArgs(job), ...shapeRunResult(job) } + // A read that fails without rejecting still arrives here: the generated client + // resolves `undefined` when it cannot read the body (a truncated response, a + // dropped connection). Logs are the one payload where empty is a real answer, + // so that has to be told apart from an empty log rather than reported as one. + const shapedLogs = + typeof logs !== 'string' || logs === LOGS_UNREADABLE + ? LOGS_UNREADABLE + : logs.trim() + ? cap(logs, true) + : 'No logs for this run.' + return shapeFlowRunTree(results, { + ...summary, + ...diagnoseRun(job), + // A successful read always carries the job itself as the root entry, so + // no entries means the read failed ā€” and nothing else would name the run. + ...(results.entries.length === 0 ? { job_id: id, steps_unavailable: true } : {}), + ...payloads, + logs: shapedLogs + }) } - // Drill-down: the server resolves the address directly (a few indexed - // lookups, no tree enumeration) and returns the single job as an entry. + return getStepResult(workspace, id, step) +} + +/** One step's result in full, addressed by step path. The server resolves the + * address directly (a few indexed lookups, no tree enumeration) and returns the + * single job as an entry. */ +async function getStepResult(workspace: string, id: string, step: string): Promise { const response = await JobService.getFlowAllResults({ workspace, id, diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index bbcb76b30c..13799cf09f 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1861,13 +1861,13 @@ export async function buildTestRunArgs( } // The string handed back to the model when a job is backgrounded. It carries the -// job id so the model can pull status/logs on demand (get_job_logs / list_runs), +// job id so the model can pull status/args/result/logs on demand (get_run / list_runs), // and tells it the completion will be reported later (notify-only wake). function backgroundedSummary(jobId: string, label: string): string { return ( `Job ${jobId} for "${label}" is taking a while and is now running in the background ā€” ` + `the chat is free to continue and you'll be told when it finishes. ` + - `To inspect it now, call get_job_logs with id="${jobId}" (or list_runs); ` + + `To inspect it now, call get_run with id="${jobId}" (or list_runs); ` + `to stop it, call cancel_job with id="${jobId}".` ) } @@ -1898,7 +1898,7 @@ export function completedJobToolStatus(job: CompletedJob): Partial { }) const summary = formatResultSummary(job.result, job.logs, job.success) - // get_flow_run_details only exists in the global/sessions chat (the same - // hosts that wire the job hooks) ā€” don't advertise it to in-editor chats. + // get_run only exists in the global/sessions chat (the same hosts that wire + // the job hooks) ā€” don't advertise it to in-editor chats. if (detachEnabled && config.contextName === 'flow' && !job.success) { return ( summary + - `\n\nFor per-step statuses and results (subflow steps included), call get_flow_run_details with id="${jobId}".` + `\n\nFor per-step statuses and results (subflow steps included), call get_run with id="${jobId}".` ) } return summary diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index 4c70c31ef4..cca8c7cdfb 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -1240,7 +1240,7 @@ setGetRuntimeLogsHandler(async ({ sessionId: callerSessionId, limit }) => { if (entries.length === 0) { return { aiResult: - 'The raw app preview is running, but it has not emitted console logs, uncaught errors, or unhandled rejections yet. If the user reported a failure, reproduce the interaction in the preview, then call get_app_runtime_logs again. For backend.() failures, call list_app_runs and then get_job_logs for the relevant job_id.', + 'The raw app preview is running, but it has not emitted console logs, uncaught errors, or unhandled rejections yet. If the user reported a failure, reproduce the interaction in the preview, then call get_app_runtime_logs again. For backend.() failures, call list_app_runs and then get_run for the relevant job_id.', uiMessage: 'No runtime logs', toolResult: 'No runtime logs' }