feat: add load more to runs page if nb of jobs >= 1000

backend/windmill-api/openapi.yaml

@@ -5856,6 +5856,8 @@ paths:
         - $ref: "#/components/parameters/ScriptExactHash"
         - $ref: "#/components/parameters/StartedBefore"
         - $ref: "#/components/parameters/StartedAfter"
+        - $ref: "#/components/parameters/CreatedBefore"
+        - $ref: "#/components/parameters/CreatedAfter"
         - $ref: "#/components/parameters/CreatedOrStartedBefore"
         - $ref: "#/components/parameters/Running"
         - $ref: "#/components/parameters/ScheduledForBeforeNow"
@@ -9055,6 +9057,20 @@ components:
       in: query
       schema:
         type: string
+    CreatedBefore:
+      name: created_before
+      description: filter on created before (inclusive) timestamp
+      in: query
+      schema:
+        type: string
+        format: date-time
+    CreatedAfter:
+      name: created_after
+      description: filter on created after (exclusive) timestamp
+      in: query
+      schema:
+        type: string
+        format: date-time
     StartedBefore:
       name: started_before
       description: filter on started before (inclusive) timestamp

backend/windmill-api/src/jobs.rs

@@ -1161,16 +1161,10 @@ pub fn filter_list_queue_query(
         sqlb.and_where_eq("parent_job", "?".bind(pj));
     }
     if let Some(dt) = &lq.started_before {
-        sqlb.and_where_le(
-            "started_at",
-            format!("to_timestamp({}  / 1000.0)", dt.timestamp_millis()),
-        );
+        sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339()));
     }
     if let Some(dt) = &lq.started_after {
-        sqlb.and_where_ge(
-            "started_at",
-            format!("to_timestamp({}  / 1000.0)", dt.timestamp_millis()),
-        );
+        sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339()));
     }
     if let Some(fs) = &lq.is_flow_step {
         sqlb.and_where_eq("is_flow_step", fs);
@@ -1182,16 +1176,10 @@ pub fn filter_list_queue_query(
     }
 
     if let Some(dt) = &lq.created_before {
-        sqlb.and_where_le(
-            "created_at",
-            format!("to_timestamp({}  / 1000.0)", dt.timestamp_millis()),
-        );
+        sqlb.and_where_le("created_at", "?".bind(&dt.to_rfc3339()));
     }
     if let Some(dt) = &lq.created_after {
-        sqlb.and_where_ge(
-            "created_at",
-            format!("to_timestamp({}  / 1000.0)", dt.timestamp_millis()),
-        );
+        sqlb.and_where_ge("created_at", "?".bind(&dt.to_rfc3339()));
     }
 
     if let Some(dt) = &lq.created_or_started_after {
@@ -4701,36 +4689,28 @@ pub fn filter_list_completed_query(
         sqlb.and_where_eq("parent_job", "?".bind(pj));
     }
     if let Some(dt) = &lq.started_before {
-        sqlb.and_where_le(
-            "started_at",
-            format!("to_timestamp({}  / 1000.0)", dt.timestamp_millis()),
-        );
+        sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339()));
     }
     if let Some(dt) = &lq.started_after {
-        sqlb.and_where_ge(
-            "started_at",
-            format!("to_timestamp({}  / 1000.0)", dt.timestamp_millis()),
-        );
+        sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339()));
     }
 
     if let Some(dt) = &lq.created_or_started_before {
-        sqlb.and_where_le(
-            "started_at",
-            format!("to_timestamp({}  / 1000.0)", dt.timestamp_millis()),
-        );
+        sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339()));
     }
     if let Some(dt) = &lq.created_or_started_after {
-        sqlb.and_where_ge(
-            "started_at",
-            format!("to_timestamp({}  / 1000.0)", dt.timestamp_millis()),
-        );
+        sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339()));
+    }
+
+    if let Some(dt) = &lq.created_before {
+        sqlb.and_where_le("created_at", "?".bind(&dt.to_rfc3339()));
+    }
+    if let Some(dt) = &lq.created_after {
+        sqlb.and_where_ge("created_at", "?".bind(&dt.to_rfc3339()));
     }
 
     if let Some(dt) = &lq.created_or_started_after_completed_jobs {
-        sqlb.and_where_ge(
-            "started_at",
-            format!("to_timestamp({}  / 1000.0)", dt.timestamp_millis()),
-        );
+        sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339()));
     }
 
     if let Some(sk) = &lq.is_skipped {

frontend/src/lib/components/RunChart.svelte

@@ -2,6 +2,7 @@
 	import { Scatter } from 'svelte-chartjs'
 	import 'chartjs-adapter-date-fns'
 	import zoomPlugin from 'chartjs-plugin-zoom'
+	import Tooltip2 from '$lib/components/Tooltip.svelte'
 	import {
 		Chart as ChartJS,
 		Title,
@@ -17,6 +18,7 @@
 	import type { CompletedJob } from '$lib/gen'
 	import { createEventDispatcher } from 'svelte'
 	import { getDbClockNow } from '$lib/forLater'
+	import Button from './common/button/Button.svelte'
 
 	export let jobs: CompletedJob[] | undefined = []
 	export let maxIsNow: boolean = false
@@ -24,6 +26,7 @@
 	export let maxTimeSet: string | undefined = undefined
 	export let selectedIds: string[] = []
 	export let canSelect: boolean = true
+	export let lastFetchWentToEnd: boolean = false
 
 	const dispatch = createEventDispatcher()
 	const SUCCESS_COLOR = '#4ade80'
@@ -283,5 +286,18 @@
 {maxTime} -->
 <!-- {JSON.stringify(jobs?.map((x) => x.started_at))} -->
 <div class="relative max-h-40">
+	{#if !lastFetchWentToEnd}
+		<div class="absolute top-[-10px] left-[60px]"
+			><Button
+				size="xs"
+				color="transparent"
+				variant="contained"
+				on:click={() => dispatch('loadExtra')}
+				>Load more <Tooltip2
+					>There are more jobs to load but only the first 1000 were fetched</Tooltip2
+				></Button
+			></div
+		>
+	{/if}
 	<Scatter {data} options={scatterOptions} />
 </div>

frontend/src/lib/components/ScriptBuilder.svelte

@@ -1034,6 +1034,7 @@
 				<div class="flex flex-row gap-2 grow max-w-md">
 					<div class="center-center">
 						<button
+							disabled={customUi?.topBar?.settings == false}
 							on:click={async () => {
 								metadataOpen = true
 							}}

frontend/src/lib/components/runs/JobLoader.svelte

@@ -123,10 +123,26 @@
 	}
 
 	let loadingFetch = false
+
+	export async function loadExtraJobs(): Promise<boolean> {
+		if (jobs && jobs.length > 0) {
+			const lastJob = jobs[jobs.length - 1]
+			// const minCreated = lastJob?.created_at
+			const minCreated = new Date(new Date(lastJob.created_at!).getTime() - 1).toISOString()
+
+			let olderJobs = await fetchJobs(undefined, minTs, undefined, minCreated)
+			jobs = jobs.concat(olderJobs)
+			computeCompletedJobs()
+			return olderJobs?.length < 1000
+		}
+		return false
+	}
+
 	async function fetchJobs(
 		startedBefore: string | undefined,
 		startedAfter: string | undefined,
-		startedAfterCompletedJobs: string | undefined
+		startedAfterCompletedJobs: string | undefined,
+		createdBefore: string | undefined
 	): Promise<Job[]> {
 		loadingFetch = true
 		try {
@@ -139,6 +155,7 @@
 				createdOrStartedAfterCompletedJobs: startedAfterCompletedJobs,
 				schedulePath,
 				scriptPathExact,
+				createdBefore,
 				createdBy: user === null || user === '' ? undefined : user,
 				scriptPathStart: scriptPathStart,
 				jobKinds,
@@ -264,7 +281,7 @@
 			// lookback won't be needed anymore (just filter ended_at > minTs instead
 			const extendedMinTs = subtractDaysFromDateString(minTs, lookback)
 			if (concurrencyKey == null || concurrencyKey === '') {
-				let newJobs = await fetchJobs(maxTs, undefined, extendedMinTs)
+				let newJobs = await fetchJobs(maxTs, undefined, extendedMinTs, undefined)
 				extendedJobs = { jobs: newJobs, obscured_jobs: [] } as ExtendedJobs
 
 				// Filter on minTs here and not in the backend
@@ -372,7 +389,7 @@
 					loading = true
 					let newJobs: Job[]
 					if (concurrencyKey == null || concurrencyKey === '') {
-						newJobs = await fetchJobs(maxTs, minTs ?? ts, undefined)
+						newJobs = await fetchJobs(maxTs, minTs ?? ts, undefined, undefined)
 					} else {
 						// Obscured jobs have no ids, so we have to do the full request
 						extendedJobs = await fetchExtendedJobs(concurrencyKey, maxTs, undefined, minTs ?? ts)

frontend/src/lib/components/runs/RunsTable.svelte

@@ -19,6 +19,7 @@
 	export let selectedWorkspace: string | undefined = undefined
 	export let activeLabel: string | null = null
 	// const loadMoreQuantity: number = 100
+	export let lastFetchWentToEnd = false
 
 	function getTime(job: Job): string | undefined {
 		return job['started_at'] ?? job['scheduled_for'] ?? job['created_at']
@@ -156,9 +157,12 @@
 	$: isSelectingJobsToCancel && (allSelected = selectedIds.length === cancelableJobCount)
 	$: isSelectingJobsToCancel && (cancelableJobCount = jobs?.filter(isJobCancelable).length ?? 0)
 
-	function jobCountString(jobCount: number) {
+	function jobCountString(jobCount: number | undefined, lastFetchWentToEnd: boolean): string {
+		if (jobCount === undefined) {
+			return ''
+		}
 		const jc = jobCount
-		const isTruncated = jc >= 1000
+		const isTruncated = jc >= 1000 && !lastFetchWentToEnd
 
 		return `${jc}${isTruncated ? '+' : ''} job${jc != 1 ? 's' : ''}`
 	}
@@ -201,14 +205,14 @@
 			{#if showExternalJobs && externalJobs.length > 0}
 				<div class="w-1/12 text-2xs">
 					<div class="flex flex-row">
-						{jobs && jobCountString(jobs.length + externalJobs.length)}<Tooltip
-							>{externalJobs.length} jobs obscured</Tooltip
-						>
+						{jobs
+							? jobCountString(jobs.length + externalJobs.length, lastFetchWentToEnd)
+							: ''}<Tooltip>{externalJobs.length} jobs obscured</Tooltip>
 					</div>
 				</div>
 			{:else if $workspaceStore !== 'admins' && omittedObscuredJobs}
 				<div class="w-1/12 text-2xs flex flex-row">
-					{jobs && jobCountString(jobs.length)}
+					{jobs ? jobCountString(jobs.length, lastFetchWentToEnd) : ''}
 					<Popover>
 						<AlertTriangle size={16} class="ml-0.5 text-yellow-500" />
 						<svelte:fragment slot="text">
@@ -218,9 +222,11 @@
 					</Popover>
 				</div>
 			{:else}
-				<div class="w-1/12 text-2xs">{jobs && jobCountString(jobs.length)}</div>
+				<div class="w-1/12 text-2xs"
+					>{jobs ? jobCountString(jobs.length, lastFetchWentToEnd) : ''}</div
+				>
 			{/if}
-			<div class="w-4/12 text-xs font-semibold">Timestamp</div>
+			<div class="w-4/12 text-xs font-semibold" />
 			<div class="w-4/12 text-xs font-semibold">Path</div>
 			{#if containsLabel}
 				<div class="w-3/12 text-xs font-semibold">Label</div>
@@ -297,6 +303,16 @@
 					</div>
 				{/if}
 			</div>
+			<div slot="footer"
+				>{#if !lastFetchWentToEnd && jobs && jobs.length >= 1000}
+					<button
+						class="text-xs text-blue-600 text-center w-full pb-2"
+						on:click={() => {
+							dispatch('loadExtra')
+						}}>Load next 1000 jobs</button
+					>
+				{/if}</div
+			>
 		</VirtualList>
 	{/if}
 </div>

frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte

@@ -88,6 +88,7 @@
 	let schedulePath = $page.url.searchParams.get('schedule_path') ?? undefined
 	let jobKindsCat = $page.url.searchParams.get('job_kinds') ?? 'runs'
 	let allWorkspaces = $page.url.searchParams.get('all_workspaces') == 'true' ?? false
+	let lastFetchWentToEnd = false
 
 	function loadFromQuery() {
 		path = $page.params.path
@@ -316,6 +317,7 @@
 		maxTs = undefined
 		jobs = undefined
 		completedJobs = undefined
+		lastFetchWentToEnd = false
 		selectedManualDate = 0
 		selectedIds = []
 		jobIdsToCancel = []
@@ -503,6 +505,13 @@
 		lookback = lookbackInDays
 	}
 
+	async function loadExtra() {
+		if (jobLoader) {
+			lastFetchWentToEnd = await jobLoader.loadExtraJobs()
+			console.log(lastFetchWentToEnd)
+		}
+	}
+
 	const warnJobLimitMsg =
 		'The exact number of concurrent jobs at the beginning of the time range may be incorrect as only the last 1000 jobs are taken into account: a job that was started earlier than this limit will not be taken into account'
 
@@ -739,12 +748,14 @@
 			</div>
 			{#if graph === 'RunChart'}
 				<RunChart
+					{lastFetchWentToEnd}
 					bind:selectedIds
 					canSelect={!isSelectingJobsToCancel}
 					minTimeSet={minTs}
 					maxTimeSet={maxTs}
 					maxIsNow={maxTs == undefined}
 					jobs={completedJobs}
+					on:loadExtra={loadExtra}
 					on:zoom={async (e) => {
 						minTs = e.detail.min.toISOString()
 						maxTs = e.detail.max.toISOString()
@@ -904,7 +915,7 @@
 				</div>
 				<div class="relative w-full">
 					<div class="flex gap-1 relative w-full">
-						<span class="text-xs absolute -top-4">Max datetime</span>
+						<span class="text-xs absolute -top-4">Max</span>
 						<input
 							type="text"
 							value={maxTs ? new Date(maxTs).toLocaleString() : 'zoom x axis to set max'}
@@ -936,6 +947,7 @@
 				<Button size="xs" color="light" variant="border" on:click={reset}>Reset</Button>
 				<ManuelDatePicker
 					on:loadJobs={() => {
+						lastFetchWentToEnd = false
 						jobLoader?.loadJobs(minTs, maxTs, true, true)
 					}}
 					bind:minTs
@@ -966,6 +978,8 @@
 							{isSelectingJobsToCancel}
 							bind:selectedIds
 							bind:selectedWorkspace
+							bind:lastFetchWentToEnd
+							on:loadExtra={loadExtra}
 							on:filterByPath={filterByPath}
 							on:filterByUser={filterByUser}
 							on:filterByFolder={filterByFolder}
@@ -1102,11 +1116,13 @@
 			</div>
 			{#if graph === 'RunChart'}
 				<RunChart
+					{lastFetchWentToEnd}
 					bind:selectedIds
 					canSelect={!isSelectingJobsToCancel}
 					minTimeSet={minTs}
 					maxTimeSet={maxTs}
 					maxIsNow={maxTs == undefined}
+					on:loadExtra={loadExtra}
 					jobs={completedJobs}
 					on:zoom={async (e) => {
 						minTs = e.detail.min.toISOString()
@@ -1236,7 +1252,7 @@
 			<div class="flex flex-row gap-1 w-full max-w-lg items-center">
 				<div class="relative w-full">
 					<div class="flex gap-1 relative w-full">
-						<span class="text-xs absolute -top-4">Min datetime</span>
+						<span class="text-xs absolute -top-4">Min</span>
 
 						<input
 							type="text"
@@ -1270,7 +1286,7 @@
 				</div>
 				<div class="relative w-full">
 					<div class="flex gap-1 relative w-full">
-						<span class="text-xs absolute -top-4">Max datetime</span>
+						<span class="text-xs absolute -top-4">Max</span>
 						<input
 							class="min-w-10"
 							type="text"
@@ -1303,6 +1319,7 @@
 				<Button size="xs" color="light" variant="border" on:click={reset}>Reset</Button>
 				<ManuelDatePicker
 					on:loadJobs={() => {
+						lastFetchWentToEnd = false
 						jobLoader?.loadJobs(minTs, maxTs, true, true)
 					}}
 					bind:this={manualDatePicker}
@@ -1330,6 +1347,8 @@
 				{isSelectingJobsToCancel}
 				bind:selectedIds
 				bind:selectedWorkspace
+				bind:lastFetchWentToEnd
+				on:loadExtra={loadExtra}
 				on:select={() => {
 					if (!isSelectingJobsToCancel) runDrawer.openDrawer()
 				}}