From db19c86a2d5da734fb3485056447b5bf879bd68e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 8 Jul 2024 11:41:27 +0200 Subject: [PATCH] fix: improve input history --- backend/windmill-api/openapi.yaml | 10 +- backend/windmill-api/src/inputs.rs | 41 ++++++-- backend/windmill-api/src/jobs.rs | 6 +- .../src/lib/components/SavedInputs.svelte | 93 +++++++++++-------- .../src/lib/components/runs/JobLoader.svelte | 7 +- 5 files changed, 104 insertions(+), 53 deletions(-) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0eae422d78..725071b1bf 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -7666,6 +7666,14 @@ paths: required: true schema: type: string + - name: input + in: query + schema: + type: boolean + - name: allow_large + in: query + schema: + type: boolean responses: "200": description: args @@ -9147,8 +9155,6 @@ components: type: string name: type: string - args: - type: object created_by: type: string created_at: diff --git a/backend/windmill-api/src/inputs.rs b/backend/windmill-api/src/inputs.rs index 74ff2d3028..e971056372 100644 --- a/backend/windmill-api/src/inputs.rs +++ b/backend/windmill-api/src/inputs.rs @@ -127,7 +127,7 @@ async fn get_input_history( let mut tx = user_db.begin(&authed).await?; let sql = &format!( - "select id, created_at, created_by, CASE WHEN args is null or pg_column_size(args) < 40000 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args, success from completed_job \ + "select id, created_at, created_by, 'null'::jsonb as args, success from completed_job \ where {} = $1 and job_kind = $2 and workspace_id = $3 \ order by created_at desc limit $4 offset $5", r.runnable_type.column_name() @@ -173,19 +173,48 @@ async fn get_input_history( Ok(Json(inputs)) } +#[derive(Deserialize)] +struct GetArgs { + input: Option, + allow_large: Option, +} async fn get_args_from_history_or_saved_input( authed: ApiAuthed, Extension(user_db): Extension, + Query(g): Query, Path((w_id, job_or_input_id)): Path<(String, Uuid)>, ) -> JsonResult> { let mut tx = user_db.begin(&authed).await?; - let result_o = sqlx::query_scalar!( - "SELECT args FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT args FROM input WHERE id = $1 AND workspace_id = $2", + let result_o = if let Some(input) = g.input { + if input { + sqlx::query_scalar!( + "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", + job_or_input_id, + w_id, + g.allow_large.unwrap_or(true) + ) + .fetch_optional(&mut *tx) + .await? + } else { + sqlx::query_scalar!( + "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM completed_job WHERE id = $1 AND workspace_id = $2", + job_or_input_id, + w_id, + g.allow_large.unwrap_or(true) + ) + .fetch_optional(&mut *tx) + .await? + } + } else { + sqlx::query_scalar!( + "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", job_or_input_id, - w_id + w_id, + g.allow_large.unwrap_or(true) ) .fetch_optional(&mut *tx) - .await?; + .await? + }; tx.commit().await?; @@ -206,7 +235,7 @@ async fn list_saved_inputs( let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, InputRow>( - "select id, workspace_id, runnable_id, runnable_type, name, CASE WHEN pg_column_size(args) < 40000 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args, created_at, created_by, is_public from input \ + "select id, workspace_id, runnable_id, runnable_type, name, 'null'::jsonb as args, created_at, created_by, is_public from input \ where runnable_id = $1 and runnable_type = $2 and workspace_id = $3 \ and (is_public IS true OR created_by = $4) \ order by created_at desc limit $5 offset $6", diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 8cbf551961..6f32117fb2 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -1466,7 +1466,7 @@ async fn list_jobs( }; let sql = if lq.success.is_none() && lq.label.is_none() { - let sqlq = list_queue_jobs_query( + let mut sqlq = list_queue_jobs_query( &w_id, &ListQueueQuery { order_desc: Some(true), ..lq.into() }, UnifiedJob::queued_job_fields(), @@ -1482,10 +1482,10 @@ async fn list_jobs( offset ) } else { - sqlq.query()? + sqlq.limit(per_page).offset(offset).query()? } } else { - sqlc.unwrap().query()? + sqlc.unwrap().limit(per_page).offset(offset).query()? }; let mut tx = user_db.begin(&authed).await?; diff --git a/frontend/src/lib/components/SavedInputs.svelte b/frontend/src/lib/components/SavedInputs.svelte index a2507a3e53..d6d6407941 100644 --- a/frontend/src/lib/components/SavedInputs.svelte +++ b/frontend/src/lib/components/SavedInputs.svelte @@ -52,13 +52,21 @@ ? 'FlowPath' : undefined + let hasAlreadyFailed = false async function loadInputHistory() { - previousInputs = await InputService.getInputHistory({ - workspace: $workspaceStore!, - runnableId, - runnableType, - perPage: 10 - }) + try { + previousInputs = await InputService.getInputHistory({ + workspace: $workspaceStore!, + runnableId, + runnableType, + perPage: 10 + }) + } catch (e) { + console.error(e) + if (hasAlreadyFailed) return + hasAlreadyFailed = true + sendUserToast(`Failed to load input history: ${e}`, true) + } } async function loadSavedInputs() { @@ -139,30 +147,31 @@ } $: { - if ($workspaceStore && jobs && (scriptHash || scriptPath || flowPath)) { - console.log('loading inputs') + if ($workspaceStore && (scriptHash || scriptPath || flowPath)) { loadInputHistory() loadSavedInputs() } } + let previewArgs: any = undefined + function selectArgs(selected_args: any) { - dispatch('selected_args', selected_args) + previewArgs = selected_args } - async function loadLargeArgs(id: string | undefined) { + async function loadLargeArgs( + id: string | undefined, + input: boolean | undefined, + allowLarge: boolean + ): Promise { if (!id) return - largeArgs = await InputService.getArgsFromHistoryOrSavedInput({ + return await InputService.getArgsFromHistoryOrSavedInput({ jobOrInputId: id, - workspace: $workspaceStore! + workspace: $workspaceStore!, + input, + allowLarge }) } - - let hasLargeArgs = false - $: hasLargeArgs = - typeof selectedInput?.args === 'string' && selectedInput?.args === 'WINDMILL_TOO_BIG' - let largeArgs: any = undefined - $: hasLargeArgs && loadLargeArgs(selectedInput?.id)
@@ -207,17 +217,17 @@ {/if}
-
+
{#if savedInputs === undefined} {:else if savedInputs.length > 0} {#each savedInputs as i} {/each} + {#if jobs.length == 5} +
... there may be more runs not displayed here as the limit is 5
+ {/if} {:else}
No running runs
{/if} @@ -360,12 +376,13 @@ `w-full flex items-center justify-between gap-4 py-2 px-4 text-left border rounded-sm hover:bg-surface-hover transition-a`, selectedInput === i ? 'border-blue-500 bg-blue-50 dark:bg-blue-900' : '' )} - on:click={() => { + on:click={async () => { if (selectedInput === i) { selectedInput = null } else { selectedInput = i } + selectArgs(await loadLargeArgs(i.id, false, false)) }} >
-
+
{i.created_by}
-
Preview
- {#if hasLargeArgs} - {#if largeArgs} -
- -
- {:else} - - {/if} - {:else if Object.keys(selectedInput?.args || {}).length > 0} + {#if typeof previewArgs == 'string' && previewArgs == 'WINDMILL_TOO_BIG'} +
+ Payload too big to preview but can still be loaded
+ {:else if Object.keys(previewArgs || {}).length > 0}
- +
{:else}
diff --git a/frontend/src/lib/components/runs/JobLoader.svelte b/frontend/src/lib/components/runs/JobLoader.svelte index 66321a86db..f7f06db5e6 100644 --- a/frontend/src/lib/components/runs/JobLoader.svelte +++ b/frontend/src/lib/components/runs/JobLoader.svelte @@ -45,6 +45,7 @@ export let allWorkspaces: boolean = false export let computeMinAndMax: (() => { minTs: string; maxTs: string } | undefined) | undefined export let lookback: number = 0 + export let perPage: number | undefined = undefined let intervalId: NodeJS.Timeout | undefined let sync = true @@ -135,7 +136,8 @@ resultFilter && resultFilter != '{}' && resultFilter != '' && resultError == '' ? resultFilter : undefined, - allWorkspaces: allWorkspaces ? true : undefined + allWorkspaces: allWorkspaces ? true : undefined, + perPage }) } @@ -170,7 +172,8 @@ resultFilter && resultFilter != '{}' && resultFilter != '' && resultError == '' ? resultFilter : undefined, - allWorkspaces: allWorkspaces ? true : undefined + allWorkspaces: allWorkspaces ? true : undefined, + perPage }) }