From 9334727d99eac251b0a995916c7ea00bd9596cef Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 14 Aug 2026 13:31:26 +0200 Subject: [PATCH 01/21] feat: stream audit logs in batches when a page is slow to load (#10695) * feat: stream audit logs in batches when a page is slow to load Co-Authored-By: Claude Opus 5 (1M context) * fix: bound streamed page size and clear stale rows on stop Co-Authored-By: Claude Opus 5 (1M context) * fix: keep the runs batch cap and drop rows of a replaced query on failure Co-Authored-By: Claude Opus 5 (1M context) * fix: ignore stop once a load has settled and reset paging when one fails Co-Authored-By: Claude Opus 5 (1M context) * chore: update ee-repo-ref to aab7da6e1f8b1fadacc2208913a5d6596f06f922 This commit updates the EE repository reference after PR #727 was merged in windmill-ee-private. Previous ee-repo-ref: 59ba8d7ce9ce1de0814b159b3813c2ac2a49239a New ee-repo-ref: aab7da6e1f8b1fadacc2208913a5d6596f06f922 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-api/openapi.yaml | 9 + backend/windmill-audit/src/lib.rs | 3 + .../lib/components/BatchLoadProgress.svelte | 62 +++++ frontend/src/lib/components/RunsPage.svelte | 40 +-- .../auditLogs/AuditLogsFilters.svelte | 86 ++---- .../auditLogs/AuditLogsTable.svelte | 23 +- .../auditLogs/useAuditLogsLoader.svelte.ts | 246 ++++++++++++++++++ .../auditLogs/useAuditLogsLoader.test.ts | 19 ++ .../(root)/(logged)/audit_logs/+page.svelte | 62 ++++- 10 files changed, 438 insertions(+), 114 deletions(-) create mode 100644 frontend/src/lib/components/BatchLoadProgress.svelte create mode 100644 frontend/src/lib/components/auditLogs/useAuditLogsLoader.svelte.ts create mode 100644 frontend/src/lib/components/auditLogs/useAuditLogsLoader.test.ts diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 6f6c280541..de456c2426 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a65162b22b127b54c0686095ee1b16b04e3111f7 +aab7da6e1f8b1fadacc2208913a5d6596f06f922 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c61c092c95..df46ec1ec8 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -268,6 +268,15 @@ paths: type: string - $ref: "#/components/parameters/ResourceName" - $ref: "#/components/parameters/ActionKind" + - name: before_id + in: query + description: > + only return logs with an id strictly lower than this one. Logs are ordered by + descending id, so this is a keyset cursor to stream a page in several batches + without paying a growing offset. + schema: + type: integer + format: int64 - name: all_workspaces in: query description: get audit logs for all workspaces diff --git a/backend/windmill-audit/src/lib.rs b/backend/windmill-audit/src/lib.rs index 8e74edfbf0..f1443fdb18 100644 --- a/backend/windmill-audit/src/lib.rs +++ b/backend/windmill-audit/src/lib.rs @@ -37,5 +37,8 @@ pub struct ListAuditLogQuery { pub resource: Option, pub before: Option>, pub after: Option>, + // Keyset cursor on the `id DESC` ordering. Lets a client stream a page in small batches + // without paying a growing OFFSET on every batch. + pub before_id: Option, pub all_workspaces: Option, } diff --git a/frontend/src/lib/components/BatchLoadProgress.svelte b/frontend/src/lib/components/BatchLoadProgress.svelte new file mode 100644 index 0000000000..16cf2d7212 --- /dev/null +++ b/frontend/src/lib/components/BatchLoadProgress.svelte @@ -0,0 +1,62 @@ + + +
+ Loading {itemsLabel}: {loaded} of {total}... +
+
+
+ {#if batchSize != null} + Batch size: + { + const v = parseInt(e.currentTarget.value) + if (v >= 1 && v <= maxBatchSize) { + onBatchSizeChange?.(v) + } else { + e.currentTarget.value = String(batchSize) + } + } + }} + /> + {/if} + +
diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index d775c1d212..03b410d232 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -48,6 +48,7 @@ import { twMerge } from 'tailwind-merge' import { computeJobKinds, useJobsLoader } from '$lib/components/runs/useJobsLoader.svelte' import ConcurrentJobsChart from '$lib/components/ConcurrentJobsChart.svelte' + import BatchLoadProgress from '$lib/components/BatchLoadProgress.svelte' import { pluralize, MAX_RESOLUTION_BATCH, MAX_RESOLUTION_NOTE_LEN } from '$lib/utils' import BatchReRunOptionsPane, { type BatchReRunOptions @@ -934,35 +935,16 @@
{#if batchProgress} -
- Loading jobs: {batchProgress.loaded} of {batchProgress.total}... -
-
-
- {#if currentBatchSize != null} - Batch size: - { - const v = parseInt(e.currentTarget.value) - if (v >= 1 && v <= 1000) { - jobsLoader.restreamWithBatchSize(v) - } - }} - /> - {/if} - +
+ jobsLoader.restreamWithBatchSize(v)} + onStop={() => jobsLoader.stopBatchLoading()} + />
{/if} diff --git a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte index c367bce01c..6bf71c65fd 100644 --- a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte +++ b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte @@ -21,24 +21,21 @@ import CalendarPicker from '$lib/components/common/calendarPicker/CalendarPicker.svelte' import { type AuditLog, - AuditService, ResourceService, UserService, ScriptService, FlowService, - AppService, - CancelError + AppService } from '$lib/gen' import { userStore, workspaceStore } from '$lib/stores' import { ChevronDown, Download, Loader2, RefreshCcw } from 'lucide-svelte' - import { onDestroy, untrack } from 'svelte' + import { onDestroy, onMount, untrack } from 'svelte' import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' import Select from '../select/Select.svelte' import { usePromise } from '$lib/svelte5Utils.svelte' import { safeSelectItems } from '../select/utils.svelte' - import { CancelablePromiseUtils } from '$lib/cancelable-promise-utils' import { sendUserToast } from '$lib/toast' let usernames: string[] | undefined = $state() @@ -48,7 +45,6 @@ logs?: AuditLog[] username?: string pageIndex?: number | undefined - hasMore?: boolean before?: string | undefined after?: string | undefined perPage?: number | undefined @@ -57,13 +53,13 @@ actionKind?: ActionKind | 'all' scope?: undefined | 'all_workspaces' | 'instance' loading?: boolean + onRefresh?: () => void } let { - logs = $bindable(undefined), + logs = undefined, username = $bindable('all'), pageIndex = $bindable(1), - hasMore = $bindable(false), before = $bindable(undefined), after = $bindable(undefined), perPage = $bindable(100), @@ -71,13 +67,11 @@ resource = $bindable() as string | undefined, actionKind = $bindable(undefined), scope = $bindable(undefined), - loading = $bindable(false) + loading = false, + onRefresh }: Props = $props() $effect.pre(() => { - if (logs == undefined) { - logs = [] - } if (operation == undefined) { operation = 'all' } @@ -89,47 +83,6 @@ } }) - function loadLogs() { - loading = true - - let username_ = username == 'all' ? undefined : username - let operation_ = operation == 'all' || operation == '' ? undefined : operation - let actionKind_ = actionKind == 'all' ? undefined : actionKind - let resource_ = resource == 'all' || resource == '' ? undefined : resource - - let _promise = AuditService.listAuditLogs({ - workspace: scope === 'instance' ? 'global' : $workspaceStore!, - page: pageIndex, - perPage, - before, - after, - username: username_, - operation: operation_, - resource: resource_, - actionKind: actionKind_, - allWorkspaces: scope === 'all_workspaces' - }) - let promise = CancelablePromiseUtils.map(_promise, (value) => { - logs = value - hasMore = !logs || (logs.length > 0 && logs.length === perPage) - loading = false - }) - promise = CancelablePromiseUtils.onTimeout(promise, 4000, () => { - sendUserToast( - 'Loading audit logs is taking longer than expected...', - 'warning', - perPage > 25 - ? [{ label: 'Reduce to 25 items per page', callback: () => (perPage = 25) }] - : [] - ) - }) - promise = CancelablePromiseUtils.catchErr(promise, (e) => { - if (e instanceof CancelError) return CancelablePromiseUtils.pure(undefined) - return CancelablePromiseUtils.err(e) - }) - return promise - } - async function loadUsers() { usernames = $userStore?.is_admin || $userStore?.is_super_admin @@ -277,9 +230,6 @@ WORKSPACES_DELETE: 'workspaces.delete' } - let refresh = $state(0) - let lastRefresh = $state(-1) - function downloadAuditLogsAsJson() { if (!logs || logs.length === 0) { sendUserToast('No audit logs to download', true) @@ -302,19 +252,15 @@ URL.revokeObjectURL(url) } - // observe all the variables that should trigger an update + onMount(() => { + loadUsers() + resources.refresh() + }) + + // observe all the variables that should be reflected in the url $effect(() => { - ;[refresh, username, perPage, before, after, operation, resource, actionKind, scope, pageIndex] - return untrack(() => { - if (refresh !== lastRefresh) { - loadUsers() - resources.refresh() - lastRefresh = refresh - } - updateQueryParams() - let promise = loadLogs() - return () => promise?.cancel() - }) + ;[username, perPage, before, after, operation, resource, actionKind, scope, pageIndex] + untrack(() => updateQueryParams()) }) @@ -476,7 +422,9 @@
+ {#if batchProgress} +
+ onBatchSizeChange?.(size)} + onStop={() => onStopLoading?.()} + /> +
+ {/if}
Per page: - - +
{#if status === 'idle'} - + {#if noOAuth} +
{server.name} did not advertise OAuth support.
+ {/if} + {:else if status === 'discovering'} -
Discovering OAuth settings...
+
Checking what {server.name} supports...
{:else if status === 'discovered' && discoveryResult} -
- ✓ OAuth supported - {#if discoveryResult.supports_dynamic_registration} - (Dynamic Client Registration available) - {/if} -
- {#if discoveryResult.scopes_supported && discoveryResult.scopes_supported.length > 0} -