From 64c5590aa32e4dbff6af43e711cb6899c02e4ee3 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 7 Oct 2023 19:54:36 +0200 Subject: [PATCH] fix: improve flow status viewer for large values --- frontend/package-lock.json | 14 +- frontend/package.json | 2 +- .../src/lib/components/CenteredPage.svelte | 2 +- .../src/lib/components/DisplayResult.svelte | 63 ++++++--- .../lib/components/FlowStatusViewer.svelte | 121 ++++++++++-------- .../lib/components/flows/map/MapItem.svelte | 7 + .../src/lib/components/graph/FlowGraph.svelte | 1 + frontend/src/lib/components/graph/model.ts | 1 + frontend/src/lib/utils.ts | 36 ++++++ 9 files changed, 162 insertions(+), 85 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 2686481d19..97d1cba5be 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -103,7 +103,7 @@ "tailwindcss": "^3.3.2", "tslib": "^2.6.1", "typescript": "^5.1.3", - "vite": "^4.4.9", + "vite": "^4.4.11", "vite-plugin-monaco-editor": "^1.1.0", "yootils": "^0.3.1" }, @@ -9731,9 +9731,9 @@ } }, "node_modules/vite": { - "version": "4.4.9", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz", - "integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==", + "version": "4.4.11", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.11.tgz", + "integrity": "sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==", "dev": true, "dependencies": { "esbuild": "^0.18.10", @@ -16886,9 +16886,9 @@ } }, "vite": { - "version": "4.4.9", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz", - "integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==", + "version": "4.4.11", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.11.tgz", + "integrity": "sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==", "dev": true, "requires": { "esbuild": "^0.18.10", diff --git a/frontend/package.json b/frontend/package.json index 012cb586b5..98f87695c1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -66,7 +66,7 @@ "tailwindcss": "^3.3.2", "tslib": "^2.6.1", "typescript": "^5.1.3", - "vite": "^4.4.9", + "vite": "^4.4.11", "vite-plugin-monaco-editor": "^1.1.0", "yootils": "^0.3.1" }, diff --git a/frontend/src/lib/components/CenteredPage.svelte b/frontend/src/lib/components/CenteredPage.svelte index c715dfc63f..f92004c85d 100644 --- a/frontend/src/lib/components/CenteredPage.svelte +++ b/frontend/src/lib/components/CenteredPage.svelte @@ -3,7 +3,7 @@
-
+
diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 9fd6e3fab9..d361263226 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -2,7 +2,7 @@ import { Highlight } from 'svelte-highlight' import { json } from 'svelte-highlight/languages' import TableCustom from './TableCustom.svelte' - import { copyToClipboard, truncate } from '$lib/utils' + import { copyToClipboard, roughSizeOfObject, truncate } from '$lib/utils' import { Button, Drawer, DrawerContent } from './common' import { ClipboardCopy, Download, Expand } from 'lucide-svelte' import Portal from 'svelte-portal' @@ -62,7 +62,20 @@ return keys.map((k) => Array.isArray(result[k])).reduce((a, b) => a && b) } + let length: undefined | number = undefined + let largeObject: undefined | boolean = undefined + function inferResultKind(result: any) { + length = undefined + largeObject = undefined + + length = roughSizeOfObject(result) + largeObject = length > 10000 + + if (largeObject) { + return 'json' + } + if (result) { try { let keys = Object.keys(result) @@ -113,7 +126,10 @@ } let jsonViewer: Drawer - $: jsonStr = JSON.stringify(result, null, 4) + + function toJsonStr(result: any) { + return JSON.stringify(result, null, 4) + } function contentOrRootString(obj: string | { filename: string; content: string }) { if (typeof obj === 'string') { @@ -125,7 +141,7 @@
- {#if result != undefined} + {#if result != undefined && length != undefined && largeObject != undefined} {#if resultKind && resultKind != 'json'}
@@ -136,15 +152,17 @@ {/if} {#if typeof result == 'object' && Object.keys(result).length > 0}
- {#if !disableDetails} - The result keys are: {truncate(Object.keys(result).join(', '), 50)} - {/if} - {#if !disableExpand} -
- - -
- {/if} + {#if !disableDetails && !largeObject} + The result keys are: {truncate(Object.keys(result).join(', '), 50)} + {/if} + {#if !disableExpand} +
+ + +
+ {/if}
{/if}{#if !forceJson && resultKind == 'table-col'}
@@ -273,13 +291,13 @@ >
{:else} - {#if jsonStr.length > 10000} + {#if largeObject}
DownloadDownload JSON is too large to be displayed in full.
@@ -292,15 +310,14 @@
{:else} - + {/if} {/if} {:else} -
No result: {jsonStr}
+
No result: {toJsonStr(result)}
{/if}
- {#if !disableExpand} @@ -314,15 +331,19 @@ : `data:text/json;charset=utf-8,${encodeURIComponent(jsonStr)}`} >Download - - {#if jsonStr.length > 100000} + {#if largeObject}
DownloadDownload JSON is too large to be displayed in full.
@@ -334,7 +355,7 @@
{:else} - + {/if} diff --git a/frontend/src/lib/components/FlowStatusViewer.svelte b/frontend/src/lib/components/FlowStatusViewer.svelte index 6a721683d5..a7663d2b2e 100644 --- a/frontend/src/lib/components/FlowStatusViewer.svelte +++ b/frontend/src/lib/components/FlowStatusViewer.svelte @@ -4,7 +4,7 @@ import FlowJobResult from './FlowJobResult.svelte' import FlowPreviewStatus from './preview/FlowPreviewStatus.svelte' import Icon from 'svelte-awesome' - import { faChevronDown, faChevronUp } from '@fortawesome/free-solid-svg-icons' + import { faChevronDown, faChevronUp, faHourglassHalf } from '@fortawesome/free-solid-svg-icons' import { createEventDispatcher } from 'svelte' import { onDestroy } from 'svelte' import type { FlowState } from './flows/flowState' @@ -13,7 +13,7 @@ import Tabs from './common/tabs/Tabs.svelte' import { FlowGraph, type GraphModuleState } from './graph' import ModuleStatus from './ModuleStatus.svelte' - import { emptyString, isOwner, pluralize, truncateRev } from '$lib/utils' + import { emptyString, isOwner, msToSec, pluralize, truncateRev } from '$lib/utils' import JobArgs from './JobArgs.svelte' import { Loader2 } from 'lucide-svelte' import FlowStatusWaitingForEvents from './FlowStatusWaitingForEvents.svelte' @@ -65,11 +65,15 @@ $: { let len = (flowJobIds?.flowJobs ?? []).length if (len != lastSize) { - forloop_selected = flowJobIds?.flowJobs[len - 1] ?? '' - lastSize = len + updateForloop(len) } } + function updateForloop(len: number) { + forloop_selected = flowJobIds?.flowJobs[len - 1] ?? '' + lastSize = len + } + $: updateFailCount(job?.flow_status?.retry?.fail_count) $: suspend_status = job?.flow_status?.modules?.[job?.flow_status.step]?.count @@ -88,42 +92,44 @@ : [] ) ?? [] - $: innerModules && localFlowModuleStates && updateInnerModules() + $: innerModules && updateInnerModules() function updateInnerModules() { - innerModules.forEach((mod, i) => { - if ( - mod.type === FlowStatusModule.type.WAITING_FOR_EVENTS && - localFlowModuleStates?.[innerModules?.[i - 1]?.id ?? '']?.type == - FlowStatusModule.type.SUCCESS - ) { - localFlowModuleStates[mod.id ?? ''] = { type: mod.type, args: job?.args } - } else if ( - mod.type === FlowStatusModule.type.WAITING_FOR_EXECUTOR && - localFlowModuleStates[mod.id ?? '']?.scheduled_for == undefined - ) { - console.debug('updating', mod.job) - JobService.getJob({ - workspace: workspaceId ?? $workspaceStore ?? '', - id: mod.job ?? '' - }) - .then((job) => { - const newState = { - type: mod.type, - scheduled_for: job?.['scheduled_for'], - job_id: job?.id, - parent_module: mod['parent_module'], - args: job?.args - } - if (!deepEqual(newState, localFlowModuleStates[mod.id ?? ''])) { - localFlowModuleStates[mod.id ?? ''] = newState - } + if (localFlowModuleStates) { + innerModules.forEach((mod, i) => { + if ( + mod.type === FlowStatusModule.type.WAITING_FOR_EVENTS && + localFlowModuleStates?.[innerModules?.[i - 1]?.id ?? '']?.type == + FlowStatusModule.type.SUCCESS + ) { + localFlowModuleStates[mod.id ?? ''] = { type: mod.type, args: job?.args } + } else if ( + mod.type === FlowStatusModule.type.WAITING_FOR_EXECUTOR && + localFlowModuleStates[mod.id ?? '']?.scheduled_for == undefined + ) { + console.debug('updating', mod.job) + JobService.getJob({ + workspace: workspaceId ?? $workspaceStore ?? '', + id: mod.job ?? '' }) - .catch((e) => { - console.error(`Could not load inner module for job ${mod.job}`, e) - }) - } - }) + .then((job) => { + const newState = { + type: mod.type, + scheduled_for: job?.['scheduled_for'], + job_id: job?.id, + parent_module: mod['parent_module'], + args: job?.args + } + if (!deepEqual(newState, localFlowModuleStates[mod.id ?? ''])) { + localFlowModuleStates[mod.id ?? ''] = newState + } + }) + .catch((e) => { + console.error(`Could not load inner module for job ${mod.job}`, e) + }) + } + }) + } } let errorCount = 0 @@ -134,7 +140,7 @@ workspace: workspaceId ?? $workspaceStore ?? '', id: jobId ?? '' }) - if (JSON.stringify(newJob) !== JSON.stringify(job)) { + if (!deepEqual(job, newJob)) { job = newJob } errorCount = 0 @@ -202,6 +208,7 @@ result: job['result'], job_id: job.id, parent_module: mod['parent_module'], + duration_ms: job['duration_ms'], iteration_total: mod.iterator?.itered?.length // retries: flowState?.raw_flow } @@ -376,7 +383,8 @@ logs: 'All jobs completed', result: jobResults, job_id: e.detail.id, - iteration_total: flowJobIds?.flowJobs.length + iteration_total: flowJobIds?.flowJobs.length, + duration_ms: e.detail.duration_ms } } } @@ -515,33 +523,36 @@

No arguments

{/if} {:else if node} -
+
+ {#if node.duration_ms} + + + {msToSec(node.duration_ms)} s + + {/if} {#if node.job_id} -
- Job Id - - {truncateRev(node.job_id ?? '', 10) ?? ''} - -
+
+ + {truncateRev(node.job_id ?? '', 10)} +
{/if}
-
+
m.id === mod.id) @@ -93,6 +95,11 @@
{/if} + {#if duration_ms} +
+ {msToSec(duration_ms)}s +
+ {/if} {#if annotation && annotation != ''}
{annotation} diff --git a/frontend/src/lib/components/graph/FlowGraph.svelte b/frontend/src/lib/components/graph/FlowGraph.svelte index 505ea418a1..47cac19ba3 100644 --- a/frontend/src/lib/components/graph/FlowGraph.svelte +++ b/frontend/src/lib/components/graph/FlowGraph.svelte @@ -331,6 +331,7 @@ insertable, insertableEnd, branchable, + duration_ms: flowModuleStates?.[mod.id]?.duration_ms, bgColor: getStateColor(flowModuleStates?.[mod.id]?.type), annotation, modules, diff --git a/frontend/src/lib/components/graph/model.ts b/frontend/src/lib/components/graph/model.ts index 76d3b2674a..d7f4ba9a78 100644 --- a/frontend/src/lib/components/graph/model.ts +++ b/frontend/src/lib/components/graph/model.ts @@ -35,6 +35,7 @@ export type GraphModuleState = { parent_module?: string iteration_total?: number retries?: number + duration_ms?: number } export type NestedNodes = GraphItem[] diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index f3a28c3ad7..81bd84aeda 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -619,3 +619,39 @@ export async function tryEvery({ timeoutCode() } } + + +export function roughSizeOfObject(object: object | string) { + if (typeof object == 'string') { + return object.length * 2; + } + var objectList: any[] = []; + var stack = [ object ]; + var bytes = 0; + + while ( stack.length ) { + let value: any = stack.pop(); + + if ( typeof value === 'boolean' ) { + bytes += 4; + } + else if (typeof value === 'string' ) { + bytes += value.length * 2; + } + else if ( typeof value === 'number' ) { + bytes += 8; + } + else if ( + typeof value === 'object' + && objectList.indexOf(value) === -1 + ) + { + objectList.push(value); + + for( var i in value ) { + stack.push( value[ i ] ); + } + } + } + return bytes; +} \ No newline at end of file