fix: make timeline fit for high number of iterations for flows

frontend/src/lib/components/FlowStatusViewerInner.svelte

@@ -31,6 +31,7 @@
 		| {
 				moduleId: string
 				flowJobs: string[]
+				length: number
 		  }
 		| undefined = undefined
 	export let job: Job | undefined = undefined
@@ -55,6 +56,15 @@
 		}
 	}
 
+	if (flowJobIds) {
+		$durationStatuses[flowJobIds?.moduleId ?? ''] = {
+			...($durationStatuses[flowJobIds?.moduleId ?? ''] ?? {}),
+			iteration_from: Math.max(flowJobIds.flowJobs.length - 20, 0),
+			iteration_total: flowJobIds?.length,
+			byJob: {}
+		}
+	}
+
 	function updateForloop(len: number) {
 		forloop_selected = flowJobIds?.flowJobs[len - 1] ?? ''
 		lastSize = len
@@ -91,7 +101,6 @@
 					mod.type === FlowStatusModule.type.WAITING_FOR_EXECUTOR &&
 					$flowModuleStates[mod.id ?? '']?.scheduled_for == undefined
 				) {
-					console.debug('updating', mod.job)
 					JobService.getJob({
 						workspace: workspaceId ?? $workspaceStore ?? '',
 						id: mod.job ?? ''
@@ -177,7 +186,7 @@
 				}
 			}
 			if ($durationStatuses[mod.id] == undefined) {
-				$durationStatuses[mod.id] = {}
+				$durationStatuses[mod.id] = { byJob: {} }
 			}
 			let started_at = job.started_at ? new Date(job.started_at).getTime() : undefined
 			if (job.type == 'QueuedJob') {
@@ -189,7 +198,7 @@
 					started_at,
 					parent_module: mod['parent_module']
 				}
-				$durationStatuses[mod.id][job.id] = {
+				$durationStatuses[mod.id].byJob[job.id] = {
 					created_at: job.created_at ? new Date(job.created_at).getTime() : undefined,
 					started_at
 				}
@@ -203,10 +212,11 @@
 					parent_module: mod['parent_module'],
 					duration_ms: job['duration_ms'],
 					started_at: started_at,
+					iteration: mod.iterator?.itered?.length,
 					iteration_total: mod.iterator?.itered?.length
 					// retries: $flowStateStore?.raw_flow
 				}
-				$durationStatuses[mod.id][job.id] = {
+				$durationStatuses[mod.id].byJob[job.id] = {
 					created_at: job.created_at ? new Date(job.created_at).getTime() : undefined,
 					started_at,
 					duration_ms: job['duration_ms']
@@ -214,7 +224,6 @@
 			}
 		}
 	}
-	let showEmbeddeds = -20
 
 	let flowTimeline: FlowTimeline
 
@@ -229,14 +238,23 @@
 			<div class="h-8" />
 		{/if} -->
 		{#if isListJob}
-			{#if (flowJobIds?.flowJobs.length ?? 0) > 20}
-				<p class="text-tertiary italic">
-					For performance reasons, only the last 20 items are shown. <button
-						class="text-primary underline"
+			{@const lenToAdd = Math.min(
+				20,
+				$durationStatuses[flowJobIds?.moduleId ?? '']?.iteration_from ?? 0
+			)}
+
+			{#if (flowJobIds?.flowJobs.length ?? 0) > 20 && lenToAdd > 0}
+				<p class="text-tertiary italic text-xs">
+					For performance reasons, only the last 20 items are shown by default <button
+						class="text-primary underline ml-4"
 						on:click={() => {
-							showEmbeddeds -= 20
+							let r = $durationStatuses[flowJobIds?.moduleId ?? '']
+							if (r.iteration_from) {
+								r.iteration_from -= lenToAdd
+								$durationStatuses = $durationStatuses
+							}
 						}}
-						>Load 20 prior
+						>Load {lenToAdd} prior
 					</button>
 				</p>
 			{/if}
@@ -306,21 +324,29 @@
 		{/if}
 		<div class={selected != 'sequence' ? 'hidden' : ''}>
 			{#if isListJob}
+				{@const lenToAdd = Math.min(
+					20,
+					$durationStatuses[flowJobIds?.moduleId ?? '']?.iteration_from ?? 0
+				)}
 				<h3 class="text-md leading-6 font-bold text-tertiary border-b mb-4">
 					Embedded flows: ({flowJobIds?.flowJobs.length} items)
 				</h3>
-				{#if (flowJobIds?.flowJobs.length ?? 0) > 20}
-					<p class="text-tertiary italic">
-						For performance reasons, only the last 20 items are shown. <button
-							class="text-primary underline"
+				{#if (flowJobIds?.flowJobs.length ?? 0) > 20 && lenToAdd > 0}
+					<p class="text-tertiary italic text-xs">
+						For performance reasons, only the last 20 items are shown by default <button
+							class="text-primary underline ml-4"
 							on:click={() => {
-								showEmbeddeds -= 20
+								let r = $durationStatuses[flowJobIds?.moduleId ?? '']
+								if (r.iteration_from) {
+									r.iteration_from -= lenToAdd
+									$durationStatuses = $durationStatuses
+								}
 							}}
-							>Load 20 prior
+							>Load {lenToAdd} prior
 						</button>
 					</p>
 				{/if}
-				{#each (flowJobIds?.flowJobs.length ?? 0) > 20 ? flowJobIds?.flowJobs?.slice(showEmbeddeds) ?? [] : flowJobIds?.flowJobs ?? [] as loopJobId, j}
+				{#each (flowJobIds?.flowJobs.length ?? 0) > 20 ? flowJobIds?.flowJobs?.slice($durationStatuses[flowJobIds?.moduleId ?? '']?.iteration_from ?? 0) ?? [] : flowJobIds?.flowJobs ?? [] as loopJobId, j (loopJobId)}
 					{#if render}
 						<Button
 							variant={forloop_selected === loopJobId ? 'contained' : 'border'}
@@ -338,10 +364,8 @@
 								}
 							}}
 						>
-							<span class="truncate">
-								#{(flowJobIds?.flowJobs.length ?? 0) > 20
-									? (flowJobIds?.flowJobs.length ?? 0) + showEmbeddeds + j + 1
-									: j + 1}: {loopJobId}
+							<span class="truncate font-mono">
+								#{($durationStatuses[flowJobIds?.moduleId ?? '']?.iteration_from ?? 0) + j + 1}: {loopJobId}
 							</span>
 
 							<Icon
@@ -386,7 +410,7 @@
 
 									let job_id = e.detail.id
 									if ($durationStatuses[modId] == undefined) {
-										$durationStatuses[modId] = {}
+										$durationStatuses[modId] = { byJob: {} }
 									}
 									if (e.detail.type == 'QueuedJob') {
 										$flowModuleStates[modId] = {
@@ -395,11 +419,12 @@
 											logs: e.detail.logs,
 											job_id,
 											args: e.detail.args,
-											iteration_total: flowJobIds?.flowJobs.length,
+											iteration: flowJobIds?.flowJobs.length,
+											iteration_total: flowJobIds?.length,
 											duration_ms: undefined
 										}
 
-										$durationStatuses[modId][job_id] = {
+										$durationStatuses[modId].byJob[job_id] = {
 											created_at,
 											started_at
 										}
@@ -413,11 +438,12 @@
 											logs: 'All jobs completed',
 											result: jobResults,
 											job_id,
-											iteration_total: flowJobIds?.flowJobs.length,
-											duration_ms: e.detail.duration_ms,
+											iteration: flowJobIds?.flowJobs.length,
+											iteration_total: flowJobIds?.length,
+											duration_ms: undefined,
 											isListJob: true
 										}
-										$durationStatuses[modId][job_id] = {
+										$durationStatuses[modId].byJob[job_id] = {
 											created_at,
 											started_at,
 											duration_ms: e.detail.duration_ms
@@ -473,7 +499,8 @@
 										flowJobIds={mod.flow_jobs
 											? {
 													moduleId: mod.id,
-													flowJobs: mod.flow_jobs
+													flowJobs: mod.flow_jobs,
+													length: mod.iterator?.itered?.length ?? mod.flow_jobs.length
 											  }
 											: undefined}
 										on:jobsLoaded={(e) => onJobsLoaded(mod, e.detail)}
@@ -545,7 +572,7 @@
 								flowDone={job?.['success'] != undefined}
 								bind:this={flowTimeline}
 								flowModules={dfs(job.raw_flow?.modules ?? [], (x) => x.id)}
-								durationStatuses={$durationStatuses}
+								{durationStatuses}
 							/>
 						{:else if rightColumnSelect == 'detail'}
 							<div class="pt-2">

frontend/src/lib/components/FlowTimeline.svelte

@@ -4,11 +4,18 @@
 	import { getDbClockNow } from '$lib/forLater'
 	import { Loader2 } from 'lucide-svelte'
 	import TimelineBar from './TimelineBar.svelte'
+	import type { Writable } from 'svelte/store'
 
 	export let flowModules: string[]
-	export let durationStatuses: Record<
-		string,
-		Record<string, { created_at?: number; started_at?: number; duration_ms?: number }>
+	export let durationStatuses: Writable<
+		Record<
+			string,
+			{
+				byJob: Record<string, { created_at?: number; started_at?: number; duration_ms?: number }>
+				iteration_from?: number
+				iteration_total?: number
+			}
+		>
 	>
 	export let flowDone = false
 
@@ -23,17 +30,22 @@
 		  >
 		| undefined = undefined
 
-	let debounced = debounce(() => computeItems(durationStatuses), 30)
-	$: flowDone != undefined && durationStatuses && debounced()
+	let debounced = debounce(() => computeItems($durationStatuses), 30)
+	$: flowDone != undefined && $durationStatuses && debounced()
 
 	export function reset() {
 		min = undefined
 		max = undefined
-		items = computeItems(durationStatuses)
+		items = computeItems($durationStatuses)
 	}
 
 	function computeItems(
-		durationStatuses: Record<string, Record<string, { started_at?: number; duration_ms?: number }>>
+		durationStatuses: Record<
+			string,
+			{
+				byJob: Record<string, { created_at?: number; started_at?: number; duration_ms?: number }>
+			}
+		>
 	): any {
 		let nmin: undefined | number = undefined
 		let nmax: undefined | number = undefined
@@ -43,7 +55,7 @@
 		let cnt = 0
 		let nitems = {}
 		Object.entries(durationStatuses).forEach(([k, o]) => {
-			Object.values(o).forEach((v) => {
+			Object.values(o.byJob).forEach((v) => {
 				cnt++
 				if (v.started_at) {
 					if (!nmin) {
@@ -67,7 +79,7 @@
 					}
 				}
 			})
-			let arr = Object.entries(o).map(([k, v]) => ({ ...v, id: k }))
+			let arr = Object.entries(o.byJob).map(([k, v]) => ({ ...v, id: k }))
 			arr.sort((x, y) => {
 				if (!x.started_at) {
 					return -1
@@ -106,7 +118,7 @@
 </script>
 
 {#if items}
-	<div class="divide-y">
+	<div class="divide-y border-b">
 		<div class="px-2 py-2 grid grid-cols-12 w-full"
 			><div />
 			<div class="col-span-11 pt-1 px-2 flex text-2xs text-secondary justify-between"
@@ -133,49 +145,68 @@
 				</div>
 			</div>
 		</div>
-		{#each Object.values(flowModules) as k}
-			<div class="px-2 py-2 grid grid-cols-12 w-full"
-				><div>{k}</div>
-				<div class="col-span-11 pt-1 px-2 flex min-h-6 w-full"
-					>{#if min && total}
-						<div class="flex flex-col gap-2 w-full">
-							{#each items?.[k] ?? [] as b}
-								{@const waitingLen = b?.created_at
-									? b.started_at
-										? b.started_at - b?.created_at
-										: b.duration_ms
-										? 0
-										: now - b?.created_at
-									: 0}
-								<div class="flex w-full">
-									<TimelineBar
-										position="left"
-										id={b?.id}
-										{total}
-										{min}
-										gray
-										started_at={b.created_at}
-										len={waitingLen < 100 ? 0 : waitingLen - 100}
-										running={b?.started_at == undefined}
-									/>
-									{#if b.started_at}
+		{#each Object.values(flowModules) as k (k)}
+			<div class="overflow-auto max-h-60 shadow-inner dark:shadow-gray-700 relative">
+				{#if ($durationStatuses?.[k]?.iteration_from ?? 0) > 0}
+					<div class="w-full flex flex-row-reverse sticky top-0">
+						<button
+							class="!text-secondary underline mr-2 text-2xs text-right whitespace-nowrap"
+							on:click={() => {
+								let r = $durationStatuses[k]
+								if (r.iteration_from) {
+									r.iteration_from -= 20
+									$durationStatuses = $durationStatuses
+								}
+							}}
+							>Viewing iterations {$durationStatuses[k].iteration_from} to {$durationStatuses[k]
+								.iteration_total}. Load more
+						</button>
+					</div>
+				{/if}
+
+				<div class="px-2 py-2 grid grid-cols-6 w-full">
+					<div>{k}</div>
+					<div class="col-span-5 flex min-h-6 w-full">
+						{#if min && total}
+							<div class="flex flex-col gap-2 w-full p-2">
+								{#each items?.[k] ?? [] as b}
+									{@const waitingLen = b?.created_at
+										? b.started_at
+											? b.started_at - b?.created_at
+											: b.duration_ms
+											? 0
+											: now - b?.created_at
+										: 0}
+									<div class="flex w-full">
 										<TimelineBar
-											position={waitingLen < 100 ? 'center' : 'right'}
+											position="left"
 											id={b?.id}
 											{total}
 											{min}
-											concat
-											started_at={b.started_at}
-											len={b.started_at ? b?.duration_ms ?? now - b?.started_at : 0}
-											running={b?.duration_ms == undefined}
+											gray
+											started_at={b.created_at}
+											len={waitingLen < 100 ? 0 : waitingLen - 100}
+											running={b?.started_at == undefined}
 										/>
-									{/if}
-								</div>
-							{/each}
-						</div>
-					{/if}</div
-				></div
-			>
+										{#if b.started_at}
+											<TimelineBar
+												position={waitingLen < 100 ? 'center' : 'right'}
+												id={b?.id}
+												{total}
+												{min}
+												concat
+												started_at={b.started_at}
+												len={b.started_at ? b?.duration_ms ?? now - b?.started_at : 0}
+												running={b?.duration_ms == undefined}
+											/>
+										{/if}
+									</div>
+								{/each}
+							</div>
+						{/if}</div
+					></div
+				>
+			</div>
 		{/each}
 	</div>
 {:else}

frontend/src/lib/components/graph/FlowGraph.svelte

@@ -380,8 +380,11 @@
 					getParentIds(parent),
 					module,
 					undefined,
-					flowModuleStates?.[module.id]?.iteration_total
-						? 'Iteration ' + flowModuleStates?.[module.id]?.iteration_total
+					flowModuleStates?.[module.id]?.iteration
+						? 'Iteration ' +
+								flowModuleStates?.[module.id]?.iteration +
+								'/' +
+								(flowModuleStates?.[module.id]?.iteration_total ?? '?')
 						: '',
 					loopDepth,
 					false,

frontend/src/lib/components/graph/model.ts

@@ -28,16 +28,25 @@ export type Branch = {
 export type GraphItem = Node | Loop | Branch
 
 export type GraphModuleStates = {
-	job_id: string,
+	job_id: string
 	states: Record<string, GraphModuleState>
 }
 
 export type FlowStatusViewerContext = {
-	flowStateStore?: Writable<FlowState>,
+	flowStateStore?: Writable<FlowState>
 	flowModuleStates: Writable<Record<string, GraphModuleState>>
 	retryStatus: Writable<Record<string, number | undefined>>
-	suspendStatus: Writable<Record<string, number | undefined>>,
-	durationStatuses: Writable<Record<string, Record<string, {created_at?: number, started_at?: number, duration_ms?: number}>>>
+	suspendStatus: Writable<Record<string, number | undefined>>
+	durationStatuses: Writable<
+		Record<
+			string,
+			{
+				iteration_from?: number
+				iteration_total?: number
+				byJob: Record<string, { created_at?: number; started_at?: number; duration_ms?: number }>
+			}
+		>
+	>
 }
 export type GraphModuleState = {
 	type: FlowStatusModule.type
@@ -47,6 +56,7 @@ export type GraphModuleState = {
 	scheduled_for?: Date
 	job_id?: string
 	parent_module?: string
+	iteration?: number
 	iteration_total?: number
 	retries?: number
 	duration_ms?: number