fix: improve flow status viewer for large values

frontend/package-lock.json

@@ -103,7 +103,7 @@
 				"tailwindcss": "^3.3.2",
 				"tslib": "^2.6.1",
 				"typescript": "^5.1.3",
-				"vite": "^4.4.9",
+				"vite": "^4.4.11",
 				"vite-plugin-monaco-editor": "^1.1.0",
 				"yootils": "^0.3.1"
 			},
@@ -9731,9 +9731,9 @@
 			}
 		},
 		"node_modules/vite": {
-			"version": "4.4.9",
-			"resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz",
-			"integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==",
+			"version": "4.4.11",
+			"resolved": "https://registry.npmjs.org/vite/-/vite-4.4.11.tgz",
+			"integrity": "sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==",
 			"dev": true,
 			"dependencies": {
 				"esbuild": "^0.18.10",
@@ -16886,9 +16886,9 @@
 			}
 		},
 		"vite": {
-			"version": "4.4.9",
-			"resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz",
-			"integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==",
+			"version": "4.4.11",
+			"resolved": "https://registry.npmjs.org/vite/-/vite-4.4.11.tgz",
+			"integrity": "sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==",
 			"dev": true,
 			"requires": {
 				"esbuild": "^0.18.10",

frontend/package.json

@@ -66,7 +66,7 @@
 		"tailwindcss": "^3.3.2",
 		"tslib": "^2.6.1",
 		"typescript": "^5.1.3",
-		"vite": "^4.4.9",
+		"vite": "^4.4.11",
 		"vite-plugin-monaco-editor": "^1.1.0",
 		"yootils": "^0.3.1"
 	},

frontend/src/lib/components/CenteredPage.svelte

@@ -3,7 +3,7 @@
 </script>
 
 <div class="pb-8 h-fit-content">
-	<div class={twMerge('max-w-6xl mx-auto px-4 sm:px-6 md:px-8 h-fit-content', $$restProps.class)}>
+	<div class={twMerge('max-w-7xl mx-auto px-4 sm:px-6 md:px-8 h-fit-content', $$restProps.class)}>
 		<slot />
 	</div>
 </div>

frontend/src/lib/components/DisplayResult.svelte

@@ -2,7 +2,7 @@
 	import { Highlight } from 'svelte-highlight'
 	import { json } from 'svelte-highlight/languages'
 	import TableCustom from './TableCustom.svelte'
-	import { copyToClipboard, truncate } from '$lib/utils'
+	import { copyToClipboard, roughSizeOfObject, truncate } from '$lib/utils'
 	import { Button, Drawer, DrawerContent } from './common'
 	import { ClipboardCopy, Download, Expand } from 'lucide-svelte'
 	import Portal from 'svelte-portal'
@@ -62,7 +62,20 @@
 		return keys.map((k) => Array.isArray(result[k])).reduce((a, b) => a && b)
 	}
 
+	let length: undefined | number = undefined
+	let largeObject: undefined | boolean = undefined
+
 	function inferResultKind(result: any) {
+		length = undefined
+		largeObject = undefined
+
+		length = roughSizeOfObject(result)
+		largeObject = length > 10000
+
+		if (largeObject) {
+			return 'json'
+		}
+
 		if (result) {
 			try {
 				let keys = Object.keys(result)
@@ -113,7 +126,10 @@
 	}
 
 	let jsonViewer: Drawer
-	$: jsonStr = JSON.stringify(result, null, 4)
+
+	function toJsonStr(result: any) {
+		return JSON.stringify(result, null, 4)
+	}
 
 	function contentOrRootString(obj: string | { filename: string; content: string }) {
 		if (typeof obj === 'string') {
@@ -125,7 +141,7 @@
 </script>
 
 <div class="inline-highlight">
-	{#if result != undefined}
+	{#if result != undefined && length != undefined && largeObject != undefined}
 		{#if resultKind && resultKind != 'json'}
 			<div class="flex flex-row w-full justify-between items-center">
 				<div class="mb-2 text-tertiary text-sm">
@@ -136,15 +152,17 @@
 		{/if}
 		{#if typeof result == 'object' && Object.keys(result).length > 0}
 			<div class="mb-2 w-full min-w-[400px] text-sm relative">
-			{#if !disableDetails}
-			The result keys are: <b>{truncate(Object.keys(result).join(', '), 50)}</b>
-			{/if}
-			{#if !disableExpand}
-				<div class="text-tertiary text-xs absolute top-5.5 right-0 inline-flex gap-2">
-					<button on:click={() => copyToClipboard(jsonStr)}><ClipboardCopy size={16} /></button>
-					<button on:click={jsonViewer.openDrawer}><Expand size={16} /></button>
-				</div>
-			{/if}
+				{#if !disableDetails && !largeObject}
+					The result keys are: <b>{truncate(Object.keys(result).join(', '), 50)}</b>
+				{/if}
+				{#if !disableExpand}
+					<div class="text-tertiary text-xs absolute top-5.5 right-0 inline-flex gap-2">
+						<button on:click={() => copyToClipboard(toJsonStr(result))}
+							><ClipboardCopy size={16} /></button
+						>
+						<button on:click={jsonViewer.openDrawer}><Expand size={16} /></button>
+					</div>
+				{/if}
 			</div>{/if}{#if !forceJson && resultKind == 'table-col'}<div
 				class="grid grid-flow-col-dense border rounded-md"
 			>
@@ -273,13 +291,13 @@
 				>
 			</div>
 		{:else}
-			{#if jsonStr.length > 10000}
+			{#if largeObject}
 				<div class="text-sm mb-2 text-tertiary">
 					<a
 						download="{filename ?? 'result'}.json"
 						href={workspaceId && jobId
 							? `/api/w/${workspaceId}/jobs_u/completed/get_result/${jobId}`
-							: `data:text/json;charset=utf-8,${encodeURIComponent(jsonStr)}`}>Download</a
+							: `data:text/json;charset=utf-8,${encodeURIComponent(toJsonStr(result))}`}>Download</a
 					>
 					JSON is too large to be displayed in full.
 				</div>
@@ -292,15 +310,14 @@
 					</Button>
 				</div>
 			{:else}
-				<Highlight language={json} code={jsonStr.replace(/\\n/g, '\n')} />
+				<Highlight language={json} code={toJsonStr(result).replace(/\\n/g, '\n')} />
 			{/if}
 		{/if}
 	{:else}
-		<div class="text-tertiary text-sm">No result: {jsonStr}</div>
+		<div class="text-tertiary text-sm">No result: {toJsonStr(result)}</div>
 	{/if}
 </div>
 
-
 {#if !disableExpand}
 	<Portal>
 		<Drawer bind:this={jsonViewer} size="900px">
@@ -314,15 +331,19 @@
 							: `data:text/json;charset=utf-8,${encodeURIComponent(jsonStr)}`}
 						>Download <Download size={14} /></a
 					>
-					<Button on:click={() => copyToClipboard(jsonStr)} color="light" size="xs">
+					<Button on:click={() => copyToClipboard(toJsonStr(result))} color="light" size="xs">
 						<div class="flex gap-2 items-center">Copy to clipboard <ClipboardCopy /> </div>
 					</Button>
 				</svelte:fragment>
-				{#if jsonStr.length > 100000}
+				{#if largeObject}
 					<div class="text-sm mb-2 text-tertiary">
 						<a
+							class="text-sm text-secondary mr-2 inline-flex gap-2 items-center py-2 px-2 hover:bg-gray-100 rounded-lg"
 							download="{filename ?? 'result'}.json"
-							href="data:text/json;charset=utf-8,{encodeURIComponent(jsonStr)}">Download</a
+							href={workspaceId && jobId
+								? `/api/w/${workspaceId}/jobs_u/completed/get_result/${jobId}`
+								: `data:text/json;charset=utf-8,${encodeURIComponent(jsonStr)}`}
+							>Download <Download size={14} /></a
 						>
 						JSON is too large to be displayed in full.
 					</div>
@@ -334,7 +355,7 @@
 						</Button>
 					</div>
 				{:else}
-					<Highlight language={json} code={jsonStr.replace(/\\n/g, '\n')} />
+					<Highlight language={json} code={toJsonStr(result).replace(/\\n/g, '\n')} />
 				{/if}
 			</DrawerContent>
 		</Drawer>

frontend/src/lib/components/FlowStatusViewer.svelte

@@ -4,7 +4,7 @@
 	import FlowJobResult from './FlowJobResult.svelte'
 	import FlowPreviewStatus from './preview/FlowPreviewStatus.svelte'
 	import Icon from 'svelte-awesome'
-	import { faChevronDown, faChevronUp } from '@fortawesome/free-solid-svg-icons'
+	import { faChevronDown, faChevronUp, faHourglassHalf } from '@fortawesome/free-solid-svg-icons'
 	import { createEventDispatcher } from 'svelte'
 	import { onDestroy } from 'svelte'
 	import type { FlowState } from './flows/flowState'
@@ -13,7 +13,7 @@
 	import Tabs from './common/tabs/Tabs.svelte'
 	import { FlowGraph, type GraphModuleState } from './graph'
 	import ModuleStatus from './ModuleStatus.svelte'
-	import { emptyString, isOwner, pluralize, truncateRev } from '$lib/utils'
+	import { emptyString, isOwner, msToSec, pluralize, truncateRev } from '$lib/utils'
 	import JobArgs from './JobArgs.svelte'
 	import { Loader2 } from 'lucide-svelte'
 	import FlowStatusWaitingForEvents from './FlowStatusWaitingForEvents.svelte'
@@ -65,11 +65,15 @@
 	$: {
 		let len = (flowJobIds?.flowJobs ?? []).length
 		if (len != lastSize) {
-			forloop_selected = flowJobIds?.flowJobs[len - 1] ?? ''
-			lastSize = len
+			updateForloop(len)
 		}
 	}
 
+	function updateForloop(len: number) {
+		forloop_selected = flowJobIds?.flowJobs[len - 1] ?? ''
+		lastSize = len
+	}
+
 	$: updateFailCount(job?.flow_status?.retry?.fail_count)
 	$: suspend_status = job?.flow_status?.modules?.[job?.flow_status.step]?.count
 
@@ -88,42 +92,44 @@
 				: []
 		) ?? []
 
-	$: innerModules && localFlowModuleStates && updateInnerModules()
+	$: innerModules && updateInnerModules()
 
 	function updateInnerModules() {
-		innerModules.forEach((mod, i) => {
-			if (
-				mod.type === FlowStatusModule.type.WAITING_FOR_EVENTS &&
-				localFlowModuleStates?.[innerModules?.[i - 1]?.id ?? '']?.type ==
-					FlowStatusModule.type.SUCCESS
-			) {
-				localFlowModuleStates[mod.id ?? ''] = { type: mod.type, args: job?.args }
-			} else if (
-				mod.type === FlowStatusModule.type.WAITING_FOR_EXECUTOR &&
-				localFlowModuleStates[mod.id ?? '']?.scheduled_for == undefined
-			) {
-				console.debug('updating', mod.job)
-				JobService.getJob({
-					workspace: workspaceId ?? $workspaceStore ?? '',
-					id: mod.job ?? ''
-				})
-					.then((job) => {
-						const newState = {
-							type: mod.type,
-							scheduled_for: job?.['scheduled_for'],
-							job_id: job?.id,
-							parent_module: mod['parent_module'],
-							args: job?.args
-						}
-						if (!deepEqual(newState, localFlowModuleStates[mod.id ?? ''])) {
-							localFlowModuleStates[mod.id ?? ''] = newState
-						}
+		if (localFlowModuleStates) {
+			innerModules.forEach((mod, i) => {
+				if (
+					mod.type === FlowStatusModule.type.WAITING_FOR_EVENTS &&
+					localFlowModuleStates?.[innerModules?.[i - 1]?.id ?? '']?.type ==
+						FlowStatusModule.type.SUCCESS
+				) {
+					localFlowModuleStates[mod.id ?? ''] = { type: mod.type, args: job?.args }
+				} else if (
+					mod.type === FlowStatusModule.type.WAITING_FOR_EXECUTOR &&
+					localFlowModuleStates[mod.id ?? '']?.scheduled_for == undefined
+				) {
+					console.debug('updating', mod.job)
+					JobService.getJob({
+						workspace: workspaceId ?? $workspaceStore ?? '',
+						id: mod.job ?? ''
 					})
-					.catch((e) => {
-						console.error(`Could not load inner module for job ${mod.job}`, e)
-					})
-			}
-		})
+						.then((job) => {
+							const newState = {
+								type: mod.type,
+								scheduled_for: job?.['scheduled_for'],
+								job_id: job?.id,
+								parent_module: mod['parent_module'],
+								args: job?.args
+							}
+							if (!deepEqual(newState, localFlowModuleStates[mod.id ?? ''])) {
+								localFlowModuleStates[mod.id ?? ''] = newState
+							}
+						})
+						.catch((e) => {
+							console.error(`Could not load inner module for job ${mod.job}`, e)
+						})
+				}
+			})
+		}
 	}
 
 	let errorCount = 0
@@ -134,7 +140,7 @@
 					workspace: workspaceId ?? $workspaceStore ?? '',
 					id: jobId ?? ''
 				})
-				if (JSON.stringify(newJob) !== JSON.stringify(job)) {
+				if (!deepEqual(job, newJob)) {
 					job = newJob
 				}
 				errorCount = 0
@@ -202,6 +208,7 @@
 					result: job['result'],
 					job_id: job.id,
 					parent_module: mod['parent_module'],
+					duration_ms: job['duration_ms'],
 					iteration_total: mod.iterator?.itered?.length
 					// retries: flowState?.raw_flow
 				}
@@ -376,7 +383,8 @@
 											logs: 'All jobs completed',
 											result: jobResults,
 											job_id: e.detail.id,
-											iteration_total: flowJobIds?.flowJobs.length
+											iteration_total: flowJobIds?.flowJobs.length,
+											duration_ms: e.detail.duration_ms
 										}
 									}
 								}
@@ -515,33 +523,36 @@
 									<p class="p-2">No arguments</p>
 								{/if}
 							{:else if node}
-								<div class="px-2 flex gap-2 min-w-0">
+								<div class="px-2 flex gap-2 min-w-0 overflow-hidden w-full">
 									<ModuleStatus type={node.type} scheduled_for={node.scheduled_for} />
+									{#if node.duration_ms}
+										<Badge>
+											<Icon data={faHourglassHalf} scale={0.6} class="mr-2" />
+											{msToSec(node.duration_ms)} s
+										</Badge>
+									{/if}
 									{#if node.job_id}
-										<div class="truncate w-full"
-											><div class=" text-primary whitespace-nowrap truncate w-full">
-												<span class="font-bold mr-2">Job Id</span>
-												<a
-													class="w-full text-right text-xs"
-													rel="noreferrer"
-													target="_blank"
-													href="/run/{node.job_id ?? ''}?workspace={job?.workspace_id}"
-												>
-													{truncateRev(node.job_id ?? '', 10) ?? ''}
-												</a>
-											</div>
+										<div class="grow w-full flex flex-row-reverse">
+											<a
+												class="text-right text-xs"
+												rel="noreferrer"
+												target="_blank"
+												href="/run/{node.job_id ?? ''}?workspace={job?.workspace_id}"
+											>
+												{truncateRev(node.job_id ?? '', 10)}
+											</a>
 										</div>
 									{/if}
 								</div>
-								<div class="px-1 border-b border-black">
+								<div class="px-1 py-1">
 									<JobArgs args={node.args} />
 								</div>
 
 								<FlowJobResult
 									workspaceId={job?.workspace_id}
-									jobId={job?.id}
-									loading={job['running'] == true}
+									jobId={node.job_id}
 									noBorder
+									loading={false}
 									col
 									result={node.result}
 									logs={node.logs ?? ''}

frontend/src/lib/components/flows/map/MapItem.svelte

@@ -11,6 +11,7 @@
 	import FlowModuleSchemaItem from './FlowModuleSchemaItem.svelte'
 	import InsertModuleButton from './InsertModuleButton.svelte'
 	import { prettyLanguage } from '$lib/common'
+	import { msToSec } from '$lib/utils'
 
 	export let mod: FlowModule
 	export let trigger: boolean
@@ -21,6 +22,7 @@
 	export let bgColor: string = ''
 	export let modules: FlowModule[]
 	export let moving: string | undefined = undefined
+	export let duration_ms: number | undefined = undefined
 
 	$: idx = modules.findIndex((m) => m.id === mod.id)
 
@@ -93,6 +95,11 @@
 			</div>
 		{/if}
 
+		{#if duration_ms}
+			<div class="absolute z-10 right-0 -top-4 center-center text-tertiary text-2xs">
+				{msToSec(duration_ms)}s
+			</div>
+		{/if}
 		{#if annotation && annotation != ''}
 			<div class="absolute z-10 left-0 -top-5 center-center text-tertiary">
 				{annotation}

frontend/src/lib/components/graph/FlowGraph.svelte

@@ -331,6 +331,7 @@
 						insertable,
 						insertableEnd,
 						branchable,
+						duration_ms: flowModuleStates?.[mod.id]?.duration_ms,
 						bgColor: getStateColor(flowModuleStates?.[mod.id]?.type),
 						annotation,
 						modules,

frontend/src/lib/components/graph/model.ts

@@ -35,6 +35,7 @@ export type GraphModuleState = {
 	parent_module?: string
 	iteration_total?: number
 	retries?: number
+	duration_ms?: number
 }
 
 export type NestedNodes = GraphItem[]

frontend/src/lib/utils.ts

@@ -619,3 +619,39 @@ export async function tryEvery({
 		timeoutCode()
 	}
 }
+
+
+export function roughSizeOfObject(object: object | string) {
+	if (typeof object == 'string') {
+		return object.length * 2;
+	}
+    var objectList: any[] = [];
+    var stack = [ object ];
+    var bytes = 0;
+
+    while ( stack.length ) {
+        let value: any = stack.pop();
+
+        if ( typeof value === 'boolean' ) {
+            bytes += 4;
+        }
+        else if (typeof value === 'string' ) {
+            bytes += value.length * 2;
+        }
+        else if ( typeof value === 'number' ) {
+            bytes += 8;
+        }
+        else if (
+            typeof value === 'object'
+            && objectList.indexOf(value) === -1
+        )
+        {
+            objectList.push(value);
+
+            for( var i in value ) {
+                stack.push( value[ i ] );
+            }
+        }
+    }
+    return bytes;
+}