From b8bf539c3fe2b4db9c74dd73f04b3029287acdc6 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 26 Aug 2026 20:24:34 +0200 Subject: [PATCH 01/40] fix(cli): keep svelte component styles in the raw-app bundle (#10838) * fix(cli): keep svelte component styles in the raw-app bundle Co-Authored-By: Claude Opus 5 * test: fold svelte style guard into the plugin test file Co-Authored-By: Claude Opus 5 * docs: record the editor-parity constraint on the svelte css option Co-Authored-By: Claude Opus 5 * test(cli): pin esbuild's service cwd before any test file chdirs esbuild's node API captures process.cwd() when its module is first imported and spawns its service with that cwd on every (re)start. createBundle stops the service after each bundle, so the cwd is reused across the whole run. Several test files chdir into a temp dir and delete it afterwards. The first one to bundle therefore pinned the service to a directory that stopped existing, and the next test to reach esbuild died with The service was stopped: ENOENT: no such file or directory, posix_spawn '.../@esbuild/linux-x64/bin/esbuild' The binary is present; ENOENT is posix_spawn rejecting the missing cwd. Which file tripped it depended on bun's readdir order, so renaming an unrelated test file was enough to surface it. Importing esbuild from the preload pins the service to a cwd that outlives the run, independent of file ordering. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01G88YF3sZFnJZUvTLVjqhZc --------- Co-authored-by: Claude Opus 5 Co-authored-by: Ruben Fiszel --- cli/src/commands/app/bundle.ts | 9 +++- ....ts => raw_app_svelte_plugin_unit.test.ts} | 44 +++++++++++++++++-- cli/test/setup.ts | 6 +++ 3 files changed, 54 insertions(+), 5 deletions(-) rename cli/test/{raw_app_svelte_module_unit.test.ts => raw_app_svelte_plugin_unit.test.ts} (76%) diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index 0ebbdaa076..286e878cfb 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -191,7 +191,14 @@ function createSveltePlugin(appDir: string): any { // Convert Svelte syntax to JavaScript try { - const { js, warnings } = svelte.compile(source, { filename }); + // The raw-app editor's in-browser bundler compiles with + // `css: "injected"`, so this must too, or the same app renders + // styled there and unstyled once the CLI builds it: Svelte's default + // ("external") hands the +`, + "styles_entry.ts": `import Styled from './Styled.svelte'; +export default Styled; +`, + }); + + const js = await bundle("styles_entry.ts"); + + const scopeClass = js.match(/

Date: Wed, 26 Aug 2026 20:46:31 +0200 Subject: [PATCH 02/40] fix: recover from unresolvable AI session links instead of a dead end (#10854) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(frontend): delete the open AI session by its stable id `session` is a $derived lookup into the session list, so it resolves to undefined as soon as the entry is dropped. Nothing reads it after the removal today, so this is latent rather than a live bug, but the delete handler is async and the id is already available as a prop that stays valid for the whole teardown. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): recover from unresolvable AI session links instead of a dead end Sessions live only in IndexedDB, keyed in the URL by `session_name`, so a link that resolves in one browser resolves to nothing in another. That hit a dead-end "Session not found" page whose only way out was a button — and it also caught a session of the user's own that had simply never been touched, since an untouched session is never persisted. Redirect instead: land on an empty session (reusing one that already exists, else creating one), replace the URL so back doesn't return to the broken link, and explain the swap in one dismissible notice above the composer. Never land on an existing conversation, which would read as a successful load. The notice explains one arrival, so it is spent the moment the arrival ends: a first message sent, the session deselected, or the page left. Deleting the open session removes it before the handler's own navigation lands — across HTTP when a fork goes with it — so that teardown is gated, otherwise recovery claims the gap and reports the session the user just deleted as missing. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .../components/sessions/SessionPicker.svelte | 47 ++++---- .../components/sessions/SessionWrapper.svelte | 105 +++++++++++++----- .../sessions/sessionRecoveryNotice.svelte.ts | 18 +++ .../sessions/sessionState.svelte.ts | 57 ++++++++-- .../components/sessions/sessionState.test.ts | 105 ++++++++++++++++++ .../(root)/(logged)/sessions/+page.svelte | 85 ++++++++++---- 6 files changed, 340 insertions(+), 77 deletions(-) create mode 100644 frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index 67b53aeff4..87462e301b 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -36,6 +36,7 @@ setNewSessionWorkspace, setSessionArchived, syncWorkspaceTo, + withOpenSessionTeardown, type Session } from './sessionState.svelte' import { unreadCountFor } from './sessionUnread.svelte' @@ -571,8 +572,8 @@ // After deleting the open session, land somewhere usable: the newest remaining // session, else a fresh one. The page derives the visible session from the - // `session_name` query, so leaving the URL on a deleted session would render - // its not-found state instead of a ready-to-type composer. + // `session_name` query, so leaving the URL on a deleted session would fall + // through to recovery and open a blank one rather than their recent work. async function openReplacementSession() { const next = sessionState.sessions[0] if (next) await activate(next) @@ -590,9 +591,11 @@ if (ids.length === 0) return const current = sessionState.currentSessionId const wasActive = !!current && ids.includes(current) - for (const id of ids) removeSession(id) - exitSelectionMode() - if (wasActive) await openReplacementSession() + await withOpenSessionTeardown(async () => { + for (const id of ids) removeSession(id) + exitSelectionMode() + if (wasActive) await openReplacementSession() + }) } async function handleConfirmedDelete() { @@ -608,23 +611,25 @@ deleteAlsoFork = false if (!session) return const wasActive = sessionState.currentSessionId === session.id - removeSession(session.id) - if (forkToDelete) { - try { - await WorkspaceService.deleteWorkspace({ workspace: forkToDelete }) - await deleteSessionsForWorkspace(forkToDelete) - sendUserToast(`Deleted forked workspace ${forkToDelete}`) - await reconcileAfterWorkspaceChange() - } catch (e: any) { - sendUserToast(`Failed to delete fork ${forkToDelete}: ${e?.body ?? e}`, true) + await withOpenSessionTeardown(async () => { + removeSession(session.id) + if (forkToDelete) { + try { + await WorkspaceService.deleteWorkspace({ workspace: forkToDelete }) + await deleteSessionsForWorkspace(forkToDelete) + sendUserToast(`Deleted forked workspace ${forkToDelete}`) + await reconcileAfterWorkspaceChange() + } catch (e: any) { + sendUserToast(`Failed to delete fork ${forkToDelete}: ${e?.body ?? e}`, true) + } } - } - // If the deleted fork was the active workspace, fall back to its parent - // so the user isn't stranded on a workspace that no longer exists. - if (forkToDelete && forkParentId && $workspaceStore === forkToDelete) { - syncWorkspaceTo(forkParentId) - } - if (wasActive) await openReplacementSession() + // If the deleted fork was the active workspace, fall back to its parent + // so the user isn't stranded on a workspace that no longer exists. + if (forkToDelete && forkParentId && $workspaceStore === forkToDelete) { + syncWorkspaceTo(forkParentId) + } + if (wasActive) await openReplacementSession() + }) } function focusAt(index: number) { diff --git a/frontend/src/lib/components/sessions/SessionWrapper.svelte b/frontend/src/lib/components/sessions/SessionWrapper.svelte index 52d9e476fb..b56b72c2be 100644 --- a/frontend/src/lib/components/sessions/SessionWrapper.svelte +++ b/frontend/src/lib/components/sessions/SessionWrapper.svelte @@ -1,5 +1,5 @@ {#snippet externalLinkHint()} @@ -315,6 +342,32 @@
Session not found
{:else} {#snippet inputPreface()} + {#if showRecoveryNotice} + + + +
+
+ + + We couldn't find that session +
+
+ {/if} {#if !hasFirstUserMessage} {/if} @@ -329,9 +382,7 @@ and reconcile would re-archive a workspace-archived one anyway. When the workspace is unavailable the SessionChangesBar below shows the move/discard banner instead (its actions are the real recovery path). --> -
+
This session is archived diff --git a/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts b/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts new file mode 100644 index 0000000000..635bd68601 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts @@ -0,0 +1,18 @@ +import { SvelteSet } from 'svelte/reactivity' + +// Session ids opened as a stand-in for a `session_name` this browser doesn't +// hold. Kept in memory rather than on the Session record: persisted, the notice +// would replay on every reload of a session the user has since made their own. +const recovered = new SvelteSet() + +export function markSessionRecovered(id: string): void { + recovered.add(id) +} + +export function isSessionRecovered(id: string): boolean { + return recovered.has(id) +} + +export function clearSessionRecovered(id: string): void { + recovered.delete(id) +} diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index d8ef56b273..505af7a43a 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -21,6 +21,7 @@ import { import { getLocalSetting, storeLocalSetting } from '$lib/utils' import { logFeatureUsage } from '$lib/utils/featureUsage' import { workspaceRootId } from './sessionScope.svelte' +import { clearSessionRecovered } from './sessionRecoveryNotice.svelte' import { type DBSchema, type IDBPDatabase } from 'idb' import { userScopedDb } from '$lib/userScopedDb' import { deleteItemsForSession } from '../copilot/chat/files/attachedFilesDB' @@ -343,8 +344,8 @@ export function setSessionDraftPrompt(sessionId: string, text: string): void { if ((s.draftPrompt ?? '') === text) return // Keep `transient` (means "in-memory only") set until the flush persists the // draft, so hydrateSessions preserves it across a reconcile inside this window; - // isReusableBlank, not `transient`, is what stops createSession reusing a typed - // draft. Only the IndexedDB write is debounced. + // isDiscardableDraft, not `transient`, is what stops createSession reusing a + // typed draft. Only the IndexedDB write is debounced. s.draftPrompt = text clearTimeout(draftPromptFlushHandles.get(sessionId)) draftPromptFlushHandles.set( @@ -748,30 +749,52 @@ export function requestComposerFocus(): void { composerFocusRequest.nonce++ } -// An untouched in-memory blank that `+` may reuse/discard. `draftPrompt === -// undefined` (never edited), not falsiness: a draft typed then erased to '' still -// has a pending flush and is a real session, so it must survive both. Every other +// An untouched in-memory blank that `+` may reuse and createSession may silently +// drop. `draftPrompt === undefined` (never edited), not falsiness: a draft typed +// then erased to '' still has a pending flush and is a real session. Every other // touch clears `transient` synchronously, so only the draft prompt needs checking. -function isReusableBlank(s: Session): boolean { +function isDiscardableDraft(s: Session): boolean { return !!s.transient && s.draftPrompt === undefined } +// Somewhere empty to put the user, for a URL naming a session this browser +// doesn't hold. Only `transient` makes "empty" trustworthy: chat seeding and +// attached-file persistence both key off `!transient` and leave every field +// below untouched, so a persisted session can hold a conversation regardless. +export function findEmptyLandingSession(): Session | undefined { + return sessionState.sessions.find( + (s) => + !!s.transient && + !s.archived && + !s.workspace_id && + // Falsiness, not `=== undefined`: we only navigate into the session, so a + // draft erased back to '' is still an empty composer to land on. + !s.draftPrompt?.trim() && + !s.pending_fork && + sessionInCurrentFamily(s) + ) +} + export function createSession(): Session { // Reuse an existing untouched draft from the active family rather than pile a // blank entry on every `+`, so several pending sessions can still be built up // in parallel, one touch at a time. A cross-family leftover blank is dropped // instead of reused (reusing it would act on that family). const reusable = sessionState.sessions.find( - (s) => isReusableBlank(s) && sessionInCurrentFamily(s) + (s) => isDiscardableDraft(s) && sessionInCurrentFamily(s) ) if (reusable) { sessionState.currentSessionId = reusable.id + // The blank recovery just landed on is exactly what this reuses, so `+` + // would otherwise hand back a session still carrying the recovery notice: + // asking for a new session must not be answered with "we couldn't find it". + clearSessionRecovered(reusable.id) // Reusing an already-active draft doesn't change currentSessionId, so ask // the composer to focus explicitly — the caller still navigates/redirects. requestComposerFocus() return reusable } - sessionState.sessions = sessionState.sessions.filter((s) => !isReusableBlank(s)) + sessionState.sessions = sessionState.sessions.filter((s) => !isDiscardableDraft(s)) const existingNumbers = sessionState.sessions .map((s) => /^session-(\d+)$/.exec(s.name)?.[1]) .map((n) => (n ? parseInt(n, 10) : 0)) @@ -1087,6 +1110,24 @@ export function setSessionArchived(id: string, archived: boolean) { persistTouched(s) } +// A counter rather than a flag: an inner teardown finishing must not reopen the +// gate while an outer one is still running. Released in a finally, so a delete +// that throws can't wedge it shut. +let openSessionTeardowns = $state(0) + +export function isTearingDownOpenSession(): boolean { + return openSessionTeardowns > 0 +} + +export async function withOpenSessionTeardown(run: () => Promise): Promise { + openSessionTeardowns++ + try { + return await run() + } finally { + openSessionTeardowns-- + } +} + export function deleteSession(id: string) { const s = sessionState.sessions.find((x) => x.id === id) if (!s) return diff --git a/frontend/src/lib/components/sessions/sessionState.test.ts b/frontend/src/lib/components/sessions/sessionState.test.ts index 7732c44746..1a54c86e78 100644 --- a/frontend/src/lib/components/sessions/sessionState.test.ts +++ b/frontend/src/lib/components/sessions/sessionState.test.ts @@ -4,14 +4,22 @@ import { commitSessionWorkspace, createSession, decideSessionLifecycle, + findEmptyLandingSession, isForkSession, + isTearingDownOpenSession, renameSession, sessionInCurrentFamily, setGeneratedSessionSummary, setSessionDraftPrompt, sessionState, + withOpenSessionTeardown, type Session } from './sessionState.svelte' +import { + clearSessionRecovered, + isSessionRecovered, + markSessionRecovered +} from './sessionRecoveryNotice.svelte' import { enterpriseLicense, usersWorkspaceStore, @@ -362,6 +370,28 @@ describe('createSession — reuses an untouched draft, family-scoped', () => { } }) + it('clears the recovery notice off the draft it reuses, so `+` is not answered with "not found"', () => { + const restore = withTwoFamilies('rootA') + const prevCurrent = sessionState.currentSessionId + const landed = session({ + id: 'recovered-blank', + name: 'session-903', + pending_workspace_id: 'forkA', + transient: true + }) + sessionState.sessions.push(landed) + markSessionRecovered(landed.id) + try { + expect(createSession().id).toBe('recovered-blank') + expect(isSessionRecovered('recovered-blank')).toBe(false) + } finally { + clearSessionRecovered('recovered-blank') + sessionState.sessions = sessionState.sessions.filter((s) => s.id !== 'recovered-blank') + sessionState.currentSessionId = prevCurrent + restore() + } + }) + it('drops an untouched draft left over from another family and starts in the active workspace', () => { const restore = withTwoFamilies('rootB') const prevCurrent = sessionState.currentSessionId @@ -488,3 +518,78 @@ describe('createSession — reuses an untouched draft, family-scoped', () => { } }) }) + +describe('findEmptyLandingSession — where an unresolvable session link lands', () => { + it('takes an untouched draft', () => { + const restore = withTwoFamilies('forkA') + const blank = session({ + id: 'landing-blank', + name: 'session-910', + pending_workspace_id: 'forkA', + transient: true + }) + sessionState.sessions.push(blank) + try { + expect(findEmptyLandingSession()?.id).toBe('landing-blank') + } finally { + sessionState.sessions = sessionState.sessions.filter((s) => s.id !== 'landing-blank') + restore() + } + }) + + it('passes over a persisted session, onto which chat seeding can graft a conversation', () => { + const restore = withTwoFamilies('forkA') + // ensureChatIdsSeeded assigns untagged legacy chats to `!transient` sessions + // and initRuntime loads them, without touching a field checked here. + const abandoned = session({ + id: 'landing-abandoned', + name: 'session-910', + pending_workspace_id: 'forkA' + }) + const others = sessionState.sessions + sessionState.sessions = [abandoned] + try { + expect(findEmptyLandingSession()).toBeUndefined() + } finally { + sessionState.sessions = others + restore() + } + }) + + it('passes over a session that has been sent, so recovery never reopens a conversation', () => { + const restore = withTwoFamilies('forkA') + const sent = session({ id: 'landing-sent', name: 'session-911', workspace_id: 'forkA' }) + // Sole candidate, so `undefined` pins the exclusion: `not.toBe` would also + // pass on any unrelated session the shared module state happens to hold. + const others = sessionState.sessions + sessionState.sessions = [sent] + try { + expect(findEmptyLandingSession()).toBeUndefined() + } finally { + sessionState.sessions = others + restore() + } + }) +}) + +describe('withOpenSessionTeardown — the gate that holds recovery off during a delete', () => { + it('stays shut until the outermost teardown finishes', async () => { + let innerDone = false + await withOpenSessionTeardown(async () => { + await withOpenSessionTeardown(async () => {}) + innerDone = true + expect(isTearingDownOpenSession()).toBe(true) + }) + expect(innerDone).toBe(true) + expect(isTearingDownOpenSession()).toBe(false) + }) + + it('reopens when the teardown throws, so a failed delete cannot wedge recovery shut', async () => { + await expect( + withOpenSessionTeardown(async () => { + throw new Error('fork deletion failed') + }) + ).rejects.toThrow('fork deletion failed') + expect(isTearingDownOpenSession()).toBe(false) + }) +}) diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index b6f5029616..fbf045fe19 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -1,5 +1,5 @@ {#if deployHub.session} + { + const target = discardTarget + discardTarget = undefined + if (target && target === deployHub.session) await target.discardUpdate() + }} + onCanceled={() => (discardTarget = undefined)} + > + + Discard this update? Everything pushed for it, including recordings made for it, is deleted + and cannot be recovered. Your published project is unaffected. + + {#key deployHub.session} {@const s = deployHub.session}
@@ -178,8 +202,10 @@
  • 1 ? 'opacity-60' : ''}> {stepNum > 1 ? '✓' : '1.'} - Bundle your project — creates a draft - on the Hub with every selected script, flow, app and resource from this folder. + Bundle your project — sends every + selected script, flow, app and resource from this folder to the Hub{s.liveOnHub + ? ' as an update' + : ' as a draft'}.
  • 2 ? 'opacity-60' : 'opacity-40'} @@ -235,7 +261,7 @@ startIcon={{ icon: Cloud }} onclick={openBundle} > - Create Hub draft ({s.selectedItems.length}) + {s.liveOnHub ? 'Bundle update' : 'Create Hub draft'} ({s.selectedItems.length}) {:else if s.phase === 'draft'} + {/if} + {/if} + {#if s.liveOnHub && s.phase === 'draft'} + {/if}
    • @@ -270,8 +319,10 @@ {#if s.phase === 'predeploy'}
    - Bundling creates a draft project on the Hub from the selected scripts, flows and - apps of {s.selectedFolder}/. + Bundling creates {s.liveOnHub + ? 'an update to your Hub project' + : 'a draft project on the Hub'} from the selected scripts, flows and apps of + {s.selectedFolder}/. {s.selectedItems.length} of {s.filteredWorkspaceItems.length} items selected.
    @@ -351,6 +402,31 @@ {/if}
    {/if} + {#if s.liveOnHub && s.phase !== 'live' && s.phase !== 'under_review'} + + Visitors keep seeing the published version, with its stars, forks and comments, + until this update is approved. Approving replaces it in place; discarding leaves it + exactly as it is. + + {/if} + {#if s.pipelineReplayMayBeStale && s.phase === 'draft'} + + This update carries the cascade recorded for the version that is live, and at least + one item has changed since. Record it again below, or visitors will replay the old + run as though it were this version. + + {/if} + {#if s.rejectionReason && s.phase === 'draft'} + + {s.rejectionReason} + + {/if} {#if s.phase === 'draft'}
    @@ -477,18 +553,21 @@
    {/if} {#if s.phase === 'under_review'} -
    - -
    - Locked while under review - - The Windmill team is reviewing this submission. Editing, recording, and sharing - actions are disabled. Estimated turnaround: 1-2 business days. - -
    -
    + The Windmill team is reviewing your project. Submission is locked until they answer + — no new version can be sent to the Hub, and no recording added to this one. + Estimated turnaround: 1-2 business days{#if s.hubSupportsUpdates}; cancel the + submission to get back to it sooner{/if}.{#if s.liveOnHub} + Visitors keep seeing the published version meanwhile, with its stars, forks and + comments; approving replaces it in place.{/if} Your folder itself is untouched — keep + editing your scripts and flows as usual. + {/if} {#if s.phase === 'draft'} {@const recordedCount = s.recordableItems.filter((i) => i.rec === 'recorded').length} @@ -668,7 +747,11 @@ Waiting for the Windmill team to review the submission. {:else} - Iterate further by starting a new draft. + + {s.liveOnHub + ? 'Publish an update to change it — this stays live until the update is approved.' + : 'Iterate further by starting a new draft.'} + {/if}
    {/snippet} diff --git a/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts index 5cc7f0d1d7..c421553a7c 100644 --- a/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts +++ b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts @@ -198,8 +198,31 @@ export class DeployToHubSession { // Whether the Hub currently has a custom logo for this project (from // rehydration) — drives the "Remove current logo" affordance. hubHasRemoteLogo = $state(false) + // A pipeline recording is attached on the Hub. An update inherits the published + // one, which is only a demo of the new version if nothing it runs changed. + hubHasPipelineRecording = $state(false) + // The Hub's own verdict: this update runs different content from the published + // version. False when there is no update in flight. + hubItemsChanged = $state(false) + // The attached pipeline recording is the published version's, copied when this + // update started, rather than one recorded for it. Authoritative across reloads, + // unlike `pipelineRecorded`, which only remembers this session. + hubPipelineRecordingInherited = $state(false) effectiveSlug = $state('') hubItemIds = $state>({}) + // Set once the project is published: everything the wizard shows from here on + // describes an update to it, and the published version keeps serving until that + // update is approved. `phase` is the update's own status, not the project's. + liveOnHub = $state(false) + /** This Hub knows about pending updates — it answers rehydration with a `live` + * key. An older one takes a project offline to republish and has neither the + * withdraw nor the discard endpoint, so the actions built on them stay hidden. */ + hubSupportsUpdates = $state(false) + // A reviewer's verdict on the current draft, shown so the publisher knows what + // to fix before resubmitting. + rejectionReason = $state(undefined) + discardingUpdate = $state(false) + withdrawing = $state(false) // Best-effort data table migrations for the bundle, editable in the drawer and // pushed on deploy. Regenerated when the bundle drawer opens. @@ -228,6 +251,10 @@ export class DeployToHubSession { submitting = $state(false) syncing = $state(false) + // Set from the Hub's answer to the draft request: this push went into an update + // rather than over the published project. + #publishedAsUpdate = false + // Intra-session tokens: latest call wins among competing calls on this session. #triggerLoadTok = 0 #recordRunTok = 0 @@ -316,6 +343,18 @@ export class DeployToHubSession { ) pipelineScriptPathSet = $derived(new Set(this.pipelineScriptPaths)) isPipelineProject = $derived(this.pipelineScriptPaths.length > 0) + /** The pipeline replay this update carries came from the published version, and + * something it runs has changed since — so it is a recording of another version. + * `hubItemsChanged` is the Hub comparing content, not a guess from which items + * carry recordings: an item nobody ever recorded has not changed. */ + pipelineReplayMayBeStale = $derived( + this.liveOnHub && + this.isPipelineProject && + this.hubHasPipelineRecording && + this.hubPipelineRecordingInherited && + this.hubItemsChanged && + !this.pipelineRecorded + ) hubSlug = $derived(this.effectiveSlug || sanitizeSlug(this.hubName)) relevantTriggers = $derived.by(() => { @@ -573,6 +612,17 @@ export class DeployToHubSession { this.hubSummary = p.summary ?? '' this.hubReadme = p.readme ?? '' this.hubHasRemoteLogo = p.has_logo === true + this.hubHasPipelineRecording = p.has_pipeline_recording === true + this.hubItemsChanged = p.items_changed === true + this.hubPipelineRecordingInherited = p.pipeline_recording_inherited === true + this.rejectionReason = p.rejection_reason ?? undefined + // `live` is a key this Hub always sends — null unless an update is in + // flight, in which case the fields above describe that update and the + // project itself is still published. Its absence means a Hub old enough to + // still take a project offline while it re-publishes, so the wizard must + // not promise otherwise. + this.hubSupportsUpdates = 'live' in p + this.liveOnHub = this.hubSupportsUpdates && (p.live?.approved === true || p.status === 'live') this.phase = p.status === 'live' ? 'live' : p.status === 'under_review' ? 'under_review' : 'draft' const ids: Record = {} @@ -898,6 +948,9 @@ export class DeployToHubSession { try { const parsed = JSON.parse(text) if (typeof parsed?.slug === 'string') returnedSlug = parsed.slug + // The Hub decides this: publishing over an approved project goes into a + // pending update instead, and the project keeps serving meanwhile. + this.#publishedAsUpdate = parsed?.pending_revision === true } catch {} if (!returnedSlug) { sendUserToast(`Hub did not return a slug. Aborting publish to avoid path drift.`, true) @@ -1252,8 +1305,13 @@ export class DeployToHubSession { // UI stuck in `predeploy`; rehydrate then upgrades to authoritative state. this.draftItems = itemsSnapshot.map((i) => ({ ...i, rec: 'none' })) this.phase = 'draft' + const asUpdate = this.#publishedAsUpdate await this.rehydrateFromHub() - sendUserToast(`Draft created on the Hub. Add recordings before submitting for review.`) + sendUserToast( + asUpdate + ? `Update ready on the Hub. Your published project stays live until it is approved.` + : `Draft created on the Hub. Add recordings before submitting for review.` + ) } finally { this.deploying = false } @@ -1313,12 +1371,82 @@ export class DeployToHubSession { } } + /** Go back to picking items, to publish again. Local only — nothing reaches the + * Hub until the bundle is confirmed, and where the Hub supports updates the + * published version keeps serving even then. */ startNewDraft = () => { this.draftItems = [] this.recordings = {} + this.rejectionReason = undefined + // All of it belongs to the update just finished, not the one starting. The + // captured cascade especially: left in place, the next update could save a + // replay of the version it replaces. Bumping the token first abandons a run + // still in flight, which would otherwise write its result back over this. + this.#pipelineRunTok++ + this.pipelineRecorded = false + this.pipelineRecordingResult = undefined + this.pipelineRunState = 'idle' + this.pipelineRunError = undefined this.phase = 'predeploy' } + /** Take the submission back out of review. Everything pushed for it is kept, so + * it can be fixed and submitted again. */ + cancelSubmission = async () => { + if (this.withdrawing) return + const slug = this.effectiveSlug + if (!slug) return + this.withdrawing = true + try { + const res = await fetch( + `/api/w/${this.workspace}/hub/projects/${encodeURIComponent(slug)}/withdraw${this.#folderQs()}`, + { method: 'POST', credentials: 'include' } + ) + if (!res.ok) { + sendUserToast(`Could not cancel the submission: ${await res.text()}`, true) + return + } + if (this.#disposed) return + this.phase = 'draft' + await this.rehydrateFromHub() + sendUserToast(`Submission cancelled. Everything you pushed is still here.`) + } catch (e: any) { + sendUserToast(`Could not cancel the submission: ${e?.message ?? e}`, true) + } finally { + this.withdrawing = false + } + } + + /** Throw away an update in progress and go back to what is published. */ + discardUpdate = async () => { + if (this.discardingUpdate) return + const slug = this.effectiveSlug + if (!slug) return + this.discardingUpdate = true + try { + const res = await fetch( + `/api/w/${this.workspace}/hub/projects/${encodeURIComponent(slug)}/discard_update${this.#folderQs()}`, + { method: 'POST', credentials: 'include' } + ) + if (!res.ok) { + sendUserToast(`Could not discard the update: ${await res.text()}`, true) + return + } + if (this.#disposed) return + this.draftItems = [] + this.recordings = {} + this.deploymentStatus = {} + this.rejectionReason = undefined + this.phase = 'live' + await this.rehydrateFromHub() + sendUserToast(`Update discarded. The published project is unchanged.`) + } catch (e: any) { + sendUserToast(`Could not discard the update: ${e?.message ?? e}`, true) + } finally { + this.discardingUpdate = false + } + } + /** Reset record-drawer state and load the target's schema. */ async openRecord(it: DeployItem) { const tok = ++this.#recordRunTok From af15a73b8b74ab2e8fa4150ffb364dc319445c52 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 26 Aug 2026 22:14:27 +0200 Subject: [PATCH 04/40] chore: move the compose stack to postgres 18 (#10827) * chore: move the compose stack to postgres 18 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ha6ovKdT9XRoj5FyVe7fEt * fix: dump the whole cluster in the postgres 18 upgrade recipe Windmill creates instance datatable, DuckLake and wm_fork_* databases in the same cluster as windmill, so a single-database pg_dump followed by removing the volume loses them silently. Dump the cluster with pg_dumpall instead, which also carries the roles the RLS policies are granted to, with their passwords. Also wait on the healthcheck before restoring, stop services generically rather than by name, and ANALYZE after the restore. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ha6ovKdT9XRoj5FyVe7fEt * fix: analyze every restored database and check the restore for errors ANALYZE is per-database, so the sibling datatable/DuckLake/wm_fork_* databases the recipe now restores were left with no planner statistics; vacuumdb --all covers them. psql does not stop on error and the old volume is gone by that point, so the restore needs an explicit grep rather than a trusted exit code. Also note that logical replication slots are never dumped, so a Postgres trigger reading a database in this cluster comes back disabled until it is re-saved. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ha6ovKdT9XRoj5FyVe7fEt * fix: drop the bootstrapped windmill database before the restore POSTGRES_DB creates an empty windmill database, so the dump's own CREATE DATABASE for it fails and its objects load into the entrypoint's database instead, keeping the new cluster's encoding and collation rather than the dumped ones. Sibling databases are created by the dump and so were never affected. Dropping it first makes the restore reproduce the source cluster exactly, and leaves one expected error instead of two. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ha6ovKdT9XRoj5FyVe7fEt * docs: move the postgres 18 upgrade runbook out of the compose file A step-by-step runbook in a config file needed corrections in three consecutive review rounds, which is the argument for keeping it somewhere it can be fixed once. The comment keeps only the constraint a reader has to know before touching the mount, plus a link. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ha6ovKdT9XRoj5FyVe7fEt * docs: point the postgres 18 upgrade note at windmill.dev GitHub gists are owned by user accounts, never organisations, so a gist is the wrong home for the only migration instructions every self-hosted operator gets. The procedure now lives in the self-host docs page instead. Depends on windmill-labs/windmilldocs#1704 merging and deploying first. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ha6ovKdT9XRoj5FyVe7fEt --------- Co-authored-by: Claude Opus 5 (1M context) --- docker-compose.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a801a0ce7c..bd1e060f22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,15 +8,28 @@ x-logging: &default-logging compress: "true" services: + ## UPGRADING FROM POSTGRES 16: db_data holds a cluster 18 cannot read, so the + ## container exits with an explanatory error rather than coming up blank. Migrating + ## means dumping the WHOLE cluster (pg_dumpall), never just the windmill database: + ## Windmill keeps datatable, DuckLake and wm_fork_* databases beside it and grants + ## its RLS policies to cluster-level roles, and a single-database dump loses both + ## silently. Full procedure, and why 16 is still a valid choice until Nov 2028: + ## https://www.windmill.dev/docs/advanced/self_host#upgrade-postgresql-to-18 db: deploy: # To use an external database, set replicas to 0 and set DATABASE_URL to the external database url in the .env file replicas: 1 - image: postgres:16 + image: postgres:18 shm_size: 1g restart: unless-stopped volumes: - - db_data:/var/lib/postgresql/data + # From 18 on the official image keeps the cluster in a major-version + # subdirectory (/var/lib/postgresql/18/docker), so the mount has to be the + # parent directory: that is what lets pg_upgrade see an old and a new + # cluster inside a single mount point. Mounting the pre-18 .../data path + # instead makes the image exit rather than start, which is what turns a + # stale 16 cluster into a loud failure instead of an empty instance. + - db_data:/var/lib/postgresql expose: - 5432 environment: From f131c3920f50f9fa18cd637eac39609495999aef Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 26 Aug 2026 23:08:36 +0200 Subject: [PATCH 05/40] fix: keep connection string query parameters under token auth (#10859) * fix: keep connection string query parameters under token auth * refactor: fold the database url parsing into one connect-options helper * docs: state the narrower invariant on base_connect_options * chore: update ee-repo-ref to 212cc7d61ec38580d4a70d9ac38d7a2cc9daf409 This commit updates the EE repository reference after PR #746 was merged in windmill-ee-private. Previous ee-repo-ref: a15d08345d7e42526c28382079ad1f575a2d1674 New ee-repo-ref: 212cc7d61ec38580d4a70d9ac38d7a2cc9daf409 Automated by sync-ee-ref workflow. --------- Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-common/src/db_params.rs | 45 ------------------------ backend/windmill-common/src/lib.rs | 19 +++++++--- 3 files changed, 15 insertions(+), 51 deletions(-) delete mode 100644 backend/windmill-common/src/db_params.rs diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 8f96f52924..bdbf08e187 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -d6aef91c0f7ba556befbf4addeb7674d4a9dd819 \ No newline at end of file +212cc7d61ec38580d4a70d9ac38d7a2cc9daf409 diff --git a/backend/windmill-common/src/db_params.rs b/backend/windmill-common/src/db_params.rs deleted file mode 100644 index d52700376d..0000000000 --- a/backend/windmill-common/src/db_params.rs +++ /dev/null @@ -1,45 +0,0 @@ -use anyhow::Result; - -/// Parsed database connection parameters, shared across DB auth providers (IAM RDS, Entra ID, etc.) -#[derive(Debug, Clone)] -pub struct DatabaseParams { - pub hostname: String, - pub port: u64, - pub username: String, - pub database: String, -} - -/// Extract database connection parameters from a PostgreSQL URL -pub fn extract_database_params(database_url: &str) -> Result { - let url = url::Url::parse(database_url) - .map_err(|e| anyhow::anyhow!("Failed to parse database URL: {}", e))?; - - let hostname = url - .host_str() - .ok_or_else(|| anyhow::anyhow!("Database URL missing hostname"))? - .to_string(); - - let port = url.port().unwrap_or(5432) as u64; - - let username = if url.username().is_empty() { - return Err(anyhow::anyhow!("Database URL missing username")); - } else { - urlencoding::decode(url.username())?.to_string() - }; - - let database = url - .path() - .trim_start_matches('/') - .split('/') - .next() - .filter(|s| !s.is_empty()) - .ok_or_else(|| anyhow::anyhow!("Database URL missing database name"))? - .to_string(); - - Ok(DatabaseParams { - hostname, - port, - username, - database: urlencoding::decode(&database)?.to_string(), - }) -} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 89db0ee769..cf0be2bbd8 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -42,7 +42,6 @@ pub mod db; mod db_entra_ee; #[cfg(all(feature = "enterprise", feature = "private"))] mod db_iam_ee; -pub mod db_params; pub mod dbt_manifest; pub mod deploy_origin; #[cfg(feature = "private")] @@ -1479,6 +1478,17 @@ pub async fn create_custom_instance_database( Ok(()) } +/// Connection options parsed from a database URL. +/// +/// The only place a database URL becomes `PgConnectOptions`. Providers that mint the password +/// themselves override it on these and keep the rest: options assembled field by field instead +/// would drop every query parameter, `sslmode` and `sslrootcert` above all, leaving the +/// connection on sqlx's default TLS policy rather than the operator's. +pub fn base_connect_options(database_url: &str) -> Result { + sqlx::postgres::PgConnectOptions::from_str(database_url) + .map_err(|e| Error::InternalErr(format!("Failed to parse database URL: {}", e))) +} + #[derive(Clone)] pub enum DatabaseUrl { #[cfg(all(feature = "enterprise", feature = "private"))] @@ -1509,8 +1519,8 @@ impl DatabaseUrl { } /// Get PgConnectOptions for this database URL. - /// For token-based auth (IAM RDS, Entra ID), this returns options built directly from the - /// token to avoid double-encoding issues with temporary credentials. + /// For token-based auth (IAM RDS, Entra ID), this returns options carrying the current + /// token, set on the builder to avoid double-encoding temporary credentials. /// For static URLs, this parses the URL string. pub async fn connect_options(&self) -> Result { match self { @@ -1524,8 +1534,7 @@ impl DatabaseUrl { let guard = entra_url.read().await; Ok(guard.connect_options()) } - DatabaseUrl::Static(url) => sqlx::postgres::PgConnectOptions::from_str(url) - .map_err(|e| Error::InternalErr(format!("Failed to parse database URL: {}", e))), + DatabaseUrl::Static(url) => base_connect_options(url), } } From 8b80b09f33d311f0881678577ca6004c12d97c22 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 26 Aug 2026 23:35:44 +0200 Subject: [PATCH 06/40] fix: restrict filesystem workspace storage to debug builds (#10864) * fix: restrict filesystem workspace storage to debug builds Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Q7p2VbtYqaXHGaAskgwVk5 * chore: update ee-repo-ref to b58ad414b098d3d7787001a352bfbb13e43a335f This commit updates the EE repository reference after PR #747 was merged in windmill-ee-private. Previous ee-repo-ref: 1b4dada77a8fe2224579c643550c63b1ac2616de New ee-repo-ref: b58ad414b098d3d7787001a352bfbb13e43a335f Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/AGENTS.md | 8 ++- backend/ee-repo-ref.txt | 2 +- .../windmill-api-workspaces/src/workspaces.rs | 17 ++++++ backend/windmill-common/src/workspaces.rs | 26 +++++++++ backend/windmill-object-store/src/lib.rs | 1 + backend/windmill-worker/src/common.rs | 1 + .../workspaceSettings/StorageSettings.svelte | 57 ++++++++++++------- 7 files changed, 87 insertions(+), 25 deletions(-) diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 0d9639c6c0..7bd84dfca3 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -131,9 +131,11 @@ minimal explicit set for dev. ## Workspace object storage in dev — use the local filesystem For a dev workspace you don't need MinIO/S3: use the built-in **`FilesystemStorage`** large-file -storage (a root path on local disk). It is intentionally hidden from the settings-UI storage -dropdown (dev-only), so set it via the API. Requires the backend built with `parquet` (+ `private` -for the real S3 helpers, + `enterprise` if you want advanced permission rules enforced): +storage (a root path on local disk). It is a **debug-build affordance only** — every site that +builds a filesystem object store calls `ensure_filesystem_storage_allowed`, so release builds +refuse it, and the settings UI never offers it — so set it via the API on a `cargo run`/`cargo +test` binary. Requires the backend built with `parquet` (+ `private` for the real S3 helpers, ++ `enterprise` if you want advanced permission rules enforced): ```bash curl -X POST "$BASE/api/w//workspaces/edit_large_file_storage_config" \ diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index bdbf08e187..4767454f25 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -212cc7d61ec38580d4a70d9ac38d7a2cc9daf409 +b58ad414b098d3d7787001a352bfbb13e43a335f diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index e04e1c1ab9..3b3ea2713c 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -1970,6 +1970,23 @@ async fn edit_large_file_storage_config( ))); } + if !windmill_common::workspaces::filesystem_storage_allowed() { + let named = std::iter::once(("primary storage", &lfs_config.large_file_storage)).chain( + lfs_config + .secondary_storage + .iter() + .map(|(name, storage)| (name.as_str(), storage)), + ); + for (name, storage) in named { + if matches!(storage, LargeFileStorage::FilesystemStorage(_)) { + return Err(Error::BadRequest(format!( + "{name}: {}", + windmill_common::workspaces::FILESYSTEM_STORAGE_DEV_ONLY_MSG + ))); + } + } + } + let serialized_lfs_config = serde_json::to_value::(lfs_config) .map_err(|err| Error::internal_err(err.to_string()))?; diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 5f558d325a..ae3e6546cd 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -2193,6 +2193,32 @@ pub fn lfs_entry_storage_ref(entry: &serde_json::Value) -> Option { Some(format!("{typ}:{path}")) } +pub const FILESYSTEM_STORAGE_DEV_ONLY_MSG: &str = + "Filesystem storage is only available in development builds of Windmill: it points the \ + workspace at a directory on the server's own disk rather than at a resource. Use an S3, \ + Azure Blob or Google Cloud Storage backend instead."; + +/// A filesystem workspace storage names a directory on the server's own disk, so it hands whoever +/// configures it — a workspace admin, or any member who can write a `filesystem` resource — +/// whatever the server process can reach, and it only resolves when server and workers share that +/// disk. It is there so local development can skip MinIO, hence debug builds only. Instance object +/// storage on local disk is a separate, superadmin-only setting and stays allowed everywhere. +pub fn filesystem_storage_allowed() -> bool { + cfg!(debug_assertions) +} + +/// Guards every site that builds an `ObjectStoreResource::Filesystem`, so nothing downstream can +/// reach a local-disk store: a stored config outlives the build that accepted it, and the resource +/// route never passes through the workspace-storage settings at all. +pub fn ensure_filesystem_storage_allowed() -> Result<()> { + if !filesystem_storage_allowed() { + return Err(Error::BadRequest( + FILESYSTEM_STORAGE_DEV_ONLY_MSG.to_string(), + )); + } + Ok(()) +} + /// Resolve a `$res:`/`$var:` reference tree to its concrete value (recursively, secrets /// decrypted). No permission checks — trusted server-side callers only; never echo the result /// to a user. diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 0f919f1048..cc588d485f 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -1171,6 +1171,7 @@ pub fn lfs_to_object_store_resource( Ok(ObjectStoreResource::Gcs(gcs_resource)) } LargeFileStorage::FilesystemStorage(fs) => { + windmill_common::workspaces::ensure_filesystem_storage_allowed()?; Ok(ObjectStoreResource::Filesystem(FilesystemSettings { root_path: fs.root_path.clone(), })) diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 6ee22fd882..f6391eab07 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1640,6 +1640,7 @@ pub(crate) async fn get_workspace_s3_resource_path( ) } Some(LargeFileStorage::FilesystemStorage(fs)) => { + windmill_common::workspaces::ensure_filesystem_storage_allowed()?; return Ok(Some( windmill_object_store::ObjectStoreResource::Filesystem( windmill_object_store::FilesystemSettings { root_path: fs.root_path.clone() }, diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte index 1e6c990819..a2ccf0b958 100644 --- a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -44,6 +44,14 @@ onDiscard?: () => void } = $props() + const creatableStorageTypes = [ + { value: 's3', label: 'S3' }, + { value: 'azure_blob', label: 'Azure Blob' }, + { value: 's3_aws_oidc', label: 'AWS OIDC' }, + { value: 'azure_workload_identity', label: 'Azure Workload Identity' }, + { value: 'gcloud_storage', label: 'Google Cloud Storage' } + ] + let advancedPermissionModalState: | { open: false } | { open: true; storage: S3ResourceSettingsItem } = $state({ open: false }) @@ -294,31 +302,38 @@
    - {#if tableRow[1].resourceType === 'filesystem'} - +
    + - {/if} + {#if tableRow[1].resourceType === 'filesystem'} + + Filesystem storage points the workspace at a directory on the server's own + disk. Only development builds of Windmill accept it — switch this storage to + S3, Azure Blob or Google Cloud Storage to configure it here. + + {/if} +
    {#if tableRow[1].resourceType === 'filesystem'} From 69320b28f615b897a92f580bd5961c41e5c29951 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 27 Aug 2026 00:39:03 +0200 Subject: [PATCH 07/40] perf: index the suspended-job resume test instead of filtering it (#10863) * perf: index the suspended-job resume test instead of filtering it Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SEUq14Wz4cC2NzcRyo6CNj * fix: keep the legacy suspended index until the replacement is recorded Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SEUq14Wz4cC2NzcRyo6CNj * perf: drop the redundant suspend_until column from the suspended index Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SEUq14Wz4cC2NzcRyo6CNj --------- Co-authored-by: Claude Opus 5 (1M context) --- ...9_queue_suspended_resume_at_index.down.sql | 1 + ...939_queue_suspended_resume_at_index.up.sql | 20 +++++ ...queue_suspended_drop_legacy_index.down.sql | 3 + ...6_queue_suspended_drop_legacy_index.up.sql | 6 ++ backend/tests/suspended_pull_index.rs | 77 +++++++++++++++++++ backend/windmill-api/src/db.rs | 8 ++ backend/windmill-common/src/worker.rs | 8 +- 7 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 backend/migrations/20260826202939_queue_suspended_resume_at_index.down.sql create mode 100644 backend/migrations/20260826202939_queue_suspended_resume_at_index.up.sql create mode 100644 backend/migrations/20260826214706_queue_suspended_drop_legacy_index.down.sql create mode 100644 backend/migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql create mode 100644 backend/tests/suspended_pull_index.rs diff --git a/backend/migrations/20260826202939_queue_suspended_resume_at_index.down.sql b/backend/migrations/20260826202939_queue_suspended_resume_at_index.down.sql new file mode 100644 index 0000000000..fe727f6761 --- /dev/null +++ b/backend/migrations/20260826202939_queue_suspended_resume_at_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS queue_suspended_v2; diff --git a/backend/migrations/20260826202939_queue_suspended_resume_at_index.up.sql b/backend/migrations/20260826202939_queue_suspended_resume_at_index.up.sql new file mode 100644 index 0000000000..6d91a64d5f --- /dev/null +++ b/backend/migrations/20260826202939_queue_suspended_resume_at_index.up.sql @@ -0,0 +1,20 @@ +-- Serves the suspended-job pull in windmill-common/src/worker.rs, whose resume test is the +-- indexed CASE expression. Two things about the shape are load-bearing: +-- * (priority DESC NULLS LAST, created_at) leads, so the scan yields that query's ORDER BY +-- and stops at the first match rather than sorting. +-- * the index is dropped before it is built rather than relying on IF NOT EXISTS. The +-- OVERRIDDEN_MIGRATIONS rewrite in windmill-api/src/db.rs runs these CONCURRENTLY, and an +-- interrupted concurrent build leaves the index present but invalid, which IF NOT EXISTS +-- would then skip rebuilding. Retiring the index this replaces is left to the migration +-- that follows, so this one can only ever be replayed while that index is still there to +-- cover the rebuild. +DROP INDEX IF EXISTS queue_suspended_v2; + +CREATE INDEX IF NOT EXISTS queue_suspended_v2 + ON v2_job_queue ( + priority DESC NULLS LAST, + created_at, + (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END), + tag + ) + WHERE suspend_until IS NOT NULL; diff --git a/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.down.sql b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.down.sql new file mode 100644 index 0000000000..011105927a --- /dev/null +++ b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.down.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS queue_suspended + ON v2_job_queue (priority DESC NULLS LAST, created_at, suspend_until, suspend, tag) + WHERE suspend_until IS NOT NULL; diff --git a/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql new file mode 100644 index 0000000000..b8851b99e0 --- /dev/null +++ b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql @@ -0,0 +1,6 @@ +-- Retires the index queue_suspended_v2 replaces. Separate from the migration that builds it +-- so that one is only ever replayed while this index still exists: sqlx records a migration +-- only after all its statements run, so a process that dies before the record is written +-- replays the build, and its leading DROP would otherwise be destroying the sole usable +-- index rather than an interrupted build. +DROP INDEX IF EXISTS queue_suspended; diff --git a/backend/tests/suspended_pull_index.rs b/backend/tests/suspended_pull_index.rs new file mode 100644 index 0000000000..4b31f4d45e --- /dev/null +++ b/backend/tests/suspended_pull_index.rs @@ -0,0 +1,77 @@ +//! Pins the plan of the suspended-job pull. Its resume test degrades silently: once the +//! query expression and `queue_suspended_v2` stop matching, Postgres still returns the right +//! job, just by falling back to a heap filter and fetching one tuple per suspended row on +//! every worker poll. No functional test can see that, so assert on the plan instead. + +use serde_json::Value; +use sqlx::{Pool, Postgres}; +use windmill_common::worker::make_suspended_pull_query; + +/// Depth-first walk of an `EXPLAIN (FORMAT JSON)` plan tree. +fn nodes(plan: &Value, out: &mut Vec) { + out.push(plan.clone()); + for child in plan["Plans"].as_array().unwrap_or(&vec![]) { + nodes(child, out); + } +} + +#[sqlx::test(fixtures("base"))] +async fn suspended_pull_tests_resume_time_inside_the_index( + db: Pool, +) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, created_at, scheduled_for, running, suspend, suspend_until, tag) + SELECT gen_random_uuid(), 'test-workspace', now() - make_interval(secs => i), + now(), true, 1 + (i % 3), now() + interval '7 day', 'flow' + FROM generate_series(1, 2000) i", + ) + .execute(&db) + .await?; + sqlx::query("ANALYZE v2_job_queue").execute(&db).await?; + + // Both plans are cheap on a 2000-row table, and which one wins there says nothing + // about a queue with a large suspended backlog. Force the index path, which is the + // one production takes, and assert on how it evaluates the resume test. + let mut conn = db.acquire().await?; + sqlx::query("SET enable_seqscan = off") + .execute(&mut *conn) + .await?; + let version: String = sqlx::query_scalar("SELECT version()") + .fetch_one(&mut *conn) + .await?; + // FORMAT JSON rather than the default: `Index Cond` and `Filter` are separate keys on the + // node, so this does not ride on EXPLAIN's line layout staying put across a major bump. + let explained: Value = sqlx::query_scalar(&format!( + "EXPLAIN (FORMAT JSON) {}", + make_suspended_pull_query(&["flow".to_string()]) + )) + .bind("test-worker") + .fetch_one(&mut *conn) + .await?; + + let mut all = vec![]; + nodes(&explained[0]["Plan"], &mut all); + let pretty = serde_json::to_string_pretty(&explained)?; + let scan = all + .iter() + .find(|n| n["Index Name"] == "queue_suspended_v2") + .unwrap_or_else(|| { + panic!("suspended pull did not scan queue_suspended_v2 on {version}:\n{pretty}") + }); + // Only `Index Cond` is checked against the index tuple, so that is where the resume test + // has to land — as a `Filter` it would cost a heap fetch per suspended row. The residual + // `suspend_until IS NOT NULL` filter is not that: it is always true for rows the partial + // index holds, and only ever runs on the row LIMIT 1 already fetched. + let cond = scan["Index Cond"].as_str().unwrap_or_else(|| { + panic!("no Index Cond on the suspended pull scan on {version}:\n{pretty}") + }); + assert!( + cond.contains("CASE WHEN"), + "resume test is not an index condition on {version}:\n{pretty}" + ); + assert!( + !scan["Filter"].as_str().unwrap_or("").contains("CASE WHEN"), + "resume test fell back to a heap filter on {version}:\n{pretty}" + ); + Ok(()) +} diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 04d21c7b53..d781e229d2 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -102,6 +102,12 @@ lazy_static::lazy_static! { (20260727151319, include_str!( "../../migrations/20260727151319_draft_only_listing_indexes.up.sql" ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY")), + (20260826202939, include_str!( + "../../migrations/20260826202939_queue_suspended_resume_at_index.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), + (20260826214706, include_str!( + "../../migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql" + ).replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), ].into_iter().collect(); } @@ -228,6 +234,8 @@ impl Migrate for CustomMigrator { // CONCURRENTLY operations cannot run inside a transaction block // or a multi-statement query (PostgreSQL requires top-level execution). // Split into individual statements and execute each separately. + // The split is naive, so a `;` anywhere in an overridden migration — + // inside a comment or a string literal included — splits mid-statement. for stmt in migration_sql.split(';') { let stmt = stmt.trim(); if !stmt.is_empty() diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 7945e73b45..84284759c6 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -734,11 +734,17 @@ fn format_pull_query(peek: String) -> String { r } +// The `CASE` is `suspend <= 0 OR suspend_until <= now()` written as one indexable +// expression, equivalent only under the `suspend_until IS NOT NULL` guard. It must stay in +// sync with `queue_suspended_v2` (migration 20260826202939): if it no longer matches, the +// test silently reverts to a heap filter over every suspended row on every worker poll. pub fn make_suspended_pull_query(tags: &[String]) -> String { format_pull_query(format!( "SELECT id FROM v2_job_queue - WHERE suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()) AND tag IN ({}) + WHERE suspend_until IS NOT NULL + AND (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END) <= now() + AND tag IN ({}) ORDER BY priority DESC NULLS LAST, created_at FOR UPDATE SKIP LOCKED LIMIT 1", From 52ca19e9aeebda821b8744a4b6ff83b26ce3a71e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 27 Aug 2026 10:15:23 +0200 Subject: [PATCH 08/40] chore(main): release 1.797.0 (#10848) * chore(main): release 1.797.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 29 +++ backend/Cargo.lock | 206 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++-- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- windmill-yaml-validator/package-lock.json | 4 +- windmill-yaml-validator/package.json | 2 +- 20 files changed, 180 insertions(+), 143 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index ae93300697..15ef7898c2 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.796.0" + ".": "1.797.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 947e63f5c6..77b3cff7d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # Changelog +## [1.797.0](https://github.com/windmill-labs/windmill/compare/v1.796.0...v1.797.0) (2026-08-26) + + +### Features + +* configurable expiry for presigned s3 public url signatures ([#10835](https://github.com/windmill-labs/windmill/issues/10835)) ([8a6dc27](https://github.com/windmill-labs/windmill/commit/8a6dc27236aca67f0efe941d9606b787c2305ea8)) +* **frontend:** flag the fork-compare datatable schema diff as legacy ([#10829](https://github.com/windmill-labs/windmill/issues/10829)) ([07c77ea](https://github.com/windmill-labs/windmill/commit/07c77ead7425f1877372d358d867445a4c525c96)) +* keep a Hub project live while an update is under review ([#10814](https://github.com/windmill-labs/windmill/issues/10814)) ([c04b570](https://github.com/windmill-labs/windmill/commit/c04b5705745c36ecbb3a551ac59459218d2e3807)) + + +### Bug Fixes + +* **cli:** keep svelte component styles in the raw-app bundle ([#10838](https://github.com/windmill-labs/windmill/issues/10838)) ([b8bf539](https://github.com/windmill-labs/windmill/commit/b8bf539c3fe2b4db9c74dd73f04b3029287acdc6)) +* **debugger:** parse bun 1.4's UUID inspector token ([#10828](https://github.com/windmill-labs/windmill/issues/10828)) ([4658224](https://github.com/windmill-labs/windmill/commit/46582245926a7f8ea961bcd125a58fbfba3530cf)) +* force HTTP router rebuild on trigger-change notification ([#10849](https://github.com/windmill-labs/windmill/issues/10849)) ([ffdf17e](https://github.com/windmill-labs/windmill/commit/ffdf17ef8dc5575dd92d62d0d0ba887c1e378576)) +* **frontend:** follow the operating workspace in step input forms ([#10834](https://github.com/windmill-labs/windmill/issues/10834)) ([6b73145](https://github.com/windmill-labs/windmill/commit/6b73145e7220232601538b801ebc9dc73fe79bbb)) +* **frontend:** key the GitHub App installation selector on installation_id ([#10831](https://github.com/windmill-labs/windmill/issues/10831)) ([78331fd](https://github.com/windmill-labs/windmill/commit/78331fda8b290a2d9a5dd92b8362ff32c8b39432)) +* **frontend:** operator menu opens on hover, pins on click ([#10824](https://github.com/windmill-labs/windmill/issues/10824)) ([665f83e](https://github.com/windmill-labs/windmill/commit/665f83e1f438e34d006429889d51a5fb6a6b6176)) +* keep connection string query parameters under token auth ([#10859](https://github.com/windmill-labs/windmill/issues/10859)) ([f131c39](https://github.com/windmill-labs/windmill/commit/f131c3920f50f9fa18cd637eac39609495999aef)) +* migrate slack resource-connect oauth to v2 ([#10836](https://github.com/windmill-labs/windmill/issues/10836)) ([9fa8159](https://github.com/windmill-labs/windmill/commit/9fa8159ad16204cab52fd18a34a48ebf13f800f6)) +* recover from unresolvable AI session links instead of a dead end ([#10854](https://github.com/windmill-labs/windmill/issues/10854)) ([e38c449](https://github.com/windmill-labs/windmill/commit/e38c449007f27b952808cba5aa812441f2ce5946)) +* require admin on workspace tarball settings export ([#10817](https://github.com/windmill-labs/windmill/issues/10817)) ([46c363f](https://github.com/windmill-labs/windmill/commit/46c363ffa4bc72bef6b367ece4bdbeef5e0eadc9)) +* restrict filesystem workspace storage to debug builds ([#10864](https://github.com/windmill-labs/windmill/issues/10864)) ([8b80b09](https://github.com/windmill-labs/windmill/commit/8b80b09f33d311f0881678577ca6004c12d97c22)) + + +### Performance Improvements + +* index the suspended-job resume test instead of filtering it ([#10863](https://github.com/windmill-labs/windmill/issues/10863)) ([69320b2](https://github.com/windmill-labs/windmill/commit/69320b28f615b897a92f580bd5961c41e5c29951)) + ## [1.796.0](https://github.com/windmill-labs/windmill/compare/v1.795.0...v1.796.0) (2026-08-24) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 2721fc7283..fcc8daa23f 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1729,7 +1729,7 @@ dependencies = [ "clang-sys", "itertools 0.13.0", "log", - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "quote", "regex", @@ -1749,7 +1749,7 @@ dependencies = [ "clang-sys", "itertools 0.13.0", "log", - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "quote", "regex", @@ -1940,34 +1940,32 @@ dependencies = [ [[package]] name = "bon" -version = "3.9.3" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6" dependencies = [ "bon-macros", - "rustversion", ] [[package]] name = "bon-macros" -version = "3.9.3" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "ident_case", - "prettyplease", + "prettyplease 0.3.0", "proc-macro2", "quote", - "rustversion", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] name = "borsh" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88b7ea17d208c4193f2c1e6de3c35fe71f98c96982d5ced308bdcc749ff6e1f" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" dependencies = [ "borsh-derive", "bytes", @@ -1976,15 +1974,15 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f347189c62a579b8cd5f80714efa178f52e461dc2e6d701d264f5ff22e566c" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] @@ -2476,9 +2474,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", "memchr", @@ -9442,6 +9440,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.4", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -14251,9 +14259,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14663,7 +14671,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-nats", @@ -14748,7 +14756,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.796.0" +version = "1.797.0" dependencies = [ "async-stream", "async-trait", @@ -14781,7 +14789,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14794,7 +14802,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "argon2", @@ -14934,7 +14942,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14957,7 +14965,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14974,7 +14982,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15000,7 +15008,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.796.0" +version = "1.797.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15010,7 +15018,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15027,7 +15035,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15049,7 +15057,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15072,7 +15080,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15088,7 +15096,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15110,7 +15118,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15131,7 +15139,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15145,7 +15153,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-nats", @@ -15180,7 +15188,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15205,7 +15213,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15233,7 +15241,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15255,7 +15263,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15275,7 +15283,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15313,7 +15321,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15341,7 +15349,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.796.0" +version = "1.797.0" dependencies = [ "lazy_static", "serde", @@ -15353,7 +15361,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.796.0" +version = "1.797.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15377,7 +15385,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15391,7 +15399,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.796.0" +version = "1.797.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15426,7 +15434,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.796.0" +version = "1.797.0" dependencies = [ "chrono", "lazy_static", @@ -15440,7 +15448,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15459,7 +15467,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.796.0" +version = "1.797.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15563,7 +15571,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.796.0" +version = "1.797.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -15582,7 +15590,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.796.0" +version = "1.797.0" dependencies = [ "regex", "serde", @@ -15597,7 +15605,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15621,7 +15629,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "futures", @@ -15638,7 +15646,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.796.0" +version = "1.797.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15654,7 +15662,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -15675,7 +15683,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -15706,7 +15714,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "arc-swap", @@ -15731,7 +15739,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-stream", @@ -15765,7 +15773,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "futures", @@ -15783,7 +15791,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.796.0" +version = "1.797.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15792,7 +15800,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -15804,7 +15812,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "serde_json", @@ -15816,7 +15824,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "gosyn", @@ -15828,7 +15836,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -15840,7 +15848,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "serde_json", @@ -15852,7 +15860,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "nu-parser", @@ -15863,7 +15871,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15874,7 +15882,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15886,7 +15894,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15897,7 +15905,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-recursion", @@ -15919,7 +15927,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "serde_json", @@ -15931,7 +15939,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -15945,7 +15953,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15962,7 +15970,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -15975,7 +15983,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "serde", @@ -15987,7 +15995,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -16005,7 +16013,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16021,7 +16029,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16037,7 +16045,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -16051,7 +16059,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-recursion", @@ -16090,7 +16098,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "const_format", @@ -16130,7 +16138,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.796.0" +version = "1.797.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16141,7 +16149,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-recursion", @@ -16176,7 +16184,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16200,7 +16208,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16233,7 +16241,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16260,7 +16268,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16293,7 +16301,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16313,7 +16321,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16347,7 +16355,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16383,7 +16391,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16406,7 +16414,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16430,7 +16438,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-nats", @@ -16454,7 +16462,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16489,7 +16497,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16517,7 +16525,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-trait", @@ -16542,7 +16550,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16561,7 +16569,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-once-cell", @@ -16678,7 +16686,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.796.0" +version = "1.797.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6dd9e452b8..887b7591f6 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.796.0" +version = "1.797.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.796.0" +version = "1.797.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 2f2fafedf4..54909e961b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.796.0" +version = "1.797.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.796.0" +version = "1.797.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.796.0" +version = "1.797.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.796.0" +version = "1.797.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 826e279c39..6f91233e28 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.796.0" +version = "1.797.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 2ca23e79b7..908056bcda 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.796.0 + version: 1.797.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 55734e52e8..519ca0c803 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.796.0"; +export const VERSION = "v1.797.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 2f80eb41fc..4b2e3d659c 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.796.0"; +export const VERSION = "1.797.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6d72964274..c73a7c41e7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.796.0", + "version": "1.797.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.796.0", + "version": "1.797.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 1d984c8587..f94ff691ae 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.796.0", + "version": "1.797.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index e0b64e8222..8ae25a1e5b 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.796.0" +wmill = ">=1.797.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 5f1882d56d..6ca54afbb2 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.796.0 + version: 1.797.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index f8cb751a07..583d84a8b2 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.796.0' + ModuleVersion = '1.797.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index cc20fab3ea..7c522c5c06 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.796.0" +version = "1.797.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 767ff7ba9d..0b3fa2e0a3 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.796.0", + "version": "1.797.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 1e1ea142a9..5295f7dc8b 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.796.0", + "version": "1.797.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index d7d21894db..23df753fdd 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.796.0 +1.797.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index 5f4baec7db..72051d74d4 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.796.0", + "version": "1.797.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.796.0", + "version": "1.797.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index bea35194bd..b3e0677aad 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.796.0", + "version": "1.797.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts", From 29133398f99cd2dd5b33057ee9df4492d82e067a Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 27 Aug 2026 10:39:36 +0200 Subject: [PATCH 09/40] feat: a wizard for importing a hub project, and finishing what the import cannot (#10729) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(frontend): guided setup wizard for data tables On Cloud a data table cannot use the Windmill instance database, so a new workspace hit a dead end: an alert telling the user to go find a PostgreSQL resource somewhere else. Setting one up meant three disconnected places, and the connection could only be tested after the config had already been saved. Adds a three-step wizard (choose a database -> set it up -> name it) reached from the data tables settings page: - Supabase: signs in via the existing supabase_wizard OAuth client and creates the project from inside Windmill. Because db_pass is an input to project creation, Windmill sets the password and the user never visits a dashboard. - Your own database: picks an existing postgresql resource, or adds one with a connection string through the form that already supports it. - Windmill database: hands back to the inline row editor, since instance databases are provisioned by a superadmin. Verifying access is no longer a step the user takes: Continue runs the check and passing it is what advances the wizard, so a database that cannot create tables never reaches the workspace config. Co-Authored-By: Claude Opus 5 (1M context) * chore: pin ee-repo-ref to the Supabase provisioning endpoints Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): do not claim the database is ready when its check failed Co-Authored-By: Claude Opus 5 (1M context) * fix: address review findings on the data table wizard - The Supabase create branch advanced on `provisioning === 4` without consulting the check it had just run, so a role that cannot create tables could reach Finish. It now blocks and offers Try again. - Retrying no longer mints a fresh secret variable + resource each time: the credentials are only re-created when the password actually changed. - The generated password is captured before the create call rather than after, since a throw there can still leave a project behind. - On a failed provision the project list is refreshed, so the just-created project can be picked up from the other tab instead of provisioning a second. - Finish refuses a name that already belongs to another data table, which previously repointed it at the new database. - Secrets go to the acting user's namespace instead of a literal `u/admin/`. - The progress list no longer ticks "Created on Supabase" before the request is sent, and does not claim the database is ready when its check failed. - The wizard's resume state is cleared when it closes, so reopening after an abandoned OAuth round trip is not stuck on step 2. - The OAuth callback shares the session-storage key rather than repeating it. - SupabaseConnect uses the shared provisioning helpers instead of a fork. - Restores the doc comment displaced onto TestDataTableResourceQuery. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): simplify Alert layout and balance its vertical padding The body was rendered by two near-duplicate branches, each wrapping the text in an extra div only to hang a margin on it, and the margins disagreed: the collapsible branch spaced above with mt-2, the static one below with mb-2. Since isCollapsed defaults to true, every non-collapsible alert took the static branch, so titled alerts read as 24px of space below the text against 16px above -- visibly off-centre -- with the title and body flush against each other. Collapse both branches into one and drop the margins; the container's own padding now sets top and bottom equally, with a small gap under the title row. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): only offer Supabase when its OAuth client is configured The wizard offered the Supabase card unconditionally, so on an instance whose superadmin never configured a supabase_wizard client -- or whose backend is built without the oauth2 feature, which compiles the whole /api/oauth router out -- the card dead-ended at a 404. Gate it on listOauthConnects, the same check ApiConnectForm already makes, fetched on open so configuring the client mid-session does not require a reload. Also drop the Supabase project ref from the existing-project cards: it is an opaque identifier that means nothing outside Supabase's own dashboard URLs. Show the region instead, plus a status word when the project is not healthy, since a paused project is the one case where the connection check fails for a reason unrelated to the password. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): run the Supabase OAuth leg in a popup A full-page redirect unmounts the wizard, so anything the user does on Supabase's side -- signing in, confirming an email, browsing their dashboard -- leaves them with nothing pointing back at Windmill, and the wizard had to park its state in sessionStorage to survive the trip. Open the connect endpoint in a popup instead. The modal stays on screen throughout and the callback hands the token back through postMessage rather than navigating. The parked-state path stays as the fallback for browsers that block the popup. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): scope the connection check to the choice that produced it A failed check stayed on screen when the user switched Supabase mode or picked a different provider, so a fresh tab opened showing an error about a database it had nothing to do with. Clear the report and the error on both switches; re-clicking the tab already selected leaves an error the user is reading in place. Also polish the Supabase step: project cards get the provider-card treatment (icon, p-3, flex column) instead of a hand-rolled variant whose block layout left more padding above the name than below; form labels settle on text-emphasis; and the signup link sits under the primary button for anyone who does not have an account yet. Drop the "free" badge and the "Free on Supabase" line -- every option in the wizard is free, so neither told the user anything -- and say what the Supabase card actually does now that connecting an existing project is the default. Co-Authored-By: Claude Opus 5 (1M context) * refactor(frontend): one setup checklist and one Supabase step for every host The data table wizard, the instance database modal and the resource drawer each had their own version of the same two interactions, and they had already begun to drift: the wizard's Supabase resource shape was rebuilt by hand in the drawer, and the instance checks rendered with no notion of a step being in flight. SetupChecklist replaces LoggedWizardResult, whose only consumer was the instance modal. It adds the running state that component lacked, so a list driven by an endpoint that reports nothing until it returns still shows where it is. Both the instance checks and the Supabase provisioning stages render through it. SupabaseProjectStep owns picking or creating a project, and useSupabaseOauth owns the popup leg. Each host keeps only what is genuinely its own: the wizard saves a variable and resource then verifies the connection, the resource drawer fills in its own form. Both trigger authorization themselves, so a host can offer it a screen earlier than the step does. The lists load behind a spinner because which mode to open on depends on whether the account has projects; deciding that after rendering flipped the toggle under the user. Adds a kitchen_sink playground for the checklist so the animation and every failure position can be exercised without a backend, a superadmin, or a Supabase account. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): tidy the resource drawer around the Supabase entry point Connect Supabase was a hand-styled anchor carrying Supabase's brand hex values rather than a Button, and it sat in a row whose other controls had settled on unifiedSize md. Making it a Button meant SupabaseIcon had to satisfy IconType, so it now takes `size` (deriving height/width from it) alongside the string props its other callers pass. The manual resource form spaced every field 32px apart and WhitelistIp added another 16px of its own, which read as a gap rather than a rhythm. One gap of 16px, with the form itself given a little more separation from the description above it. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): stop Supabase resources coming up modified when first opened Resource forms fill in every unset property from the schema as soon as they render, so a postgresql resource saved without region, root_certificate_pem and use_iam_auth was dirty -- and had saved a draft -- the first time anyone looked at it. Write them with the rest of the value. SupabaseConnect also rebuilt the resource shape by hand instead of using the shared helper, which is how the pooler host format ended up in two places. Co-Authored-By: Claude Opus 5 (1M context) * feat(backend): record where a data table came from and whether setup finished edit_datatable_config replaces the whole datatables map and DataTable does not deny unknown fields, so anything the request omits is dropped without a word. origin and setup_incomplete would have been erased by any unrelated save; preserve_unmanaged_datatable_fields carries them -- and migrations_enabled, which had the same problem inline -- forward for entries that already exist, following renames. setup_incomplete is what lets a row be recorded before the resource it points at exists, so the wizard can write nothing until the user finishes. There is deliberately no intermediate state: the setup runs entirely in the browser, so nothing server-side could advance one. datatable_health probes every data table at once for the settings page and skips the incomplete ones, whose resource_path resolves to nothing yet. set_datatable_setup patches a single entry instead of resending the map. test_datatable_connection_value checks a connection the caller has not saved anywhere, which the wizard needs before it has written a resource. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): make destructive default and subtle buttons read red Both variants were neutral until the pointer arrived, then filled solid red: nothing marked the button as destructive until you were already on it. They now carry red text at rest, with a faded red border on default and a light red wash on hover, which is what the legacy red border style in the same file had always done. Three call sites passed color="red" alongside a design-system variant. getStyleClass returns before colour is read for accent, accent-secondary, default and subtle, so the delete-migration control, its modal confirm and the import-database button had all been rendering neutral. They pass destructive now. The dropdown variant strips the button's own border, and matched border-border-light literally -- a class the destructive style no longer contains. Co-Authored-By: Claude Opus 5 (1M context) * feat(frontend): rebuild data table setup around a read-only row The wizard gathers intent over two steps, reviews it on a third and writes nothing until Finish, so a billable Supabase project is created only once the user has seen what will happen. runSetup is also the retry: every step probes for its own result before doing anything, so running it again on a half-finished data table resumes instead of duplicating. Its steps are keyed rather than dispatched on their titles, where rewording one changed what it did. The settings row stops being an editable form with a dirty/save cycle. It carries the name, where the database came from, a health dot and two actions; everything rare moved into the gear panel, which also offers Finish setup for a data table whose wizard never completed. Manage is ExploreAssetButton, the control the ducklake list already uses, and the row and panel both link out to the underlying resource. supabaseResourceValue no longer assembles the pooler host from the region. aws-0-.pooler.supabase.com is wrong for any project Supabase allocated elsewhere, so the host, user and port come from the pooler config endpoint. Two data tables sharing one database also share _wm_migrations, which is probed unqualified, so the review step warns when the database being connected is already behind another data table. SupabaseConnect is deleted. The resource drawer uses the shared project step restricted to existing projects: creating one is a billed action and belongs in the wizard, which has somewhere to report what it did. The kitchen_sink checklist playground goes with it. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): fall back to a direct Supabase connection when the pooler cannot be read Reading a project's Supavisor config needs the database_pooling_config_read scope, which an instance's Supabase OAuth app may never have been granted. No retry recovers from that, and the wizard treated it as fatal: the user was left with an error and no way to finish connecting a project that was otherwise fine. resolveSupabaseConnection replaces the bare pooler read everywhere it happened. Asking for session pooling and failing now yields a direct connection plus the reason, which supabaseResourceValue already knew how to write. Nothing about the fallback is silent -- direct is IPv6-only, which is the whole reason session pooling is the default -- so the wizard warns on its review step and the resource drawer says so in its toast. The row is recorded before credentials are saved, so an origin claiming session pooling has to be corrected once a direct host is what gets written; the run patches it through set_datatable_setup rather than leaving the panel to report a mode nothing uses. Co-Authored-By: Claude Opus 5 (1M context) * feat(frontend): open the database behind a data table, and say when it cannot write Every database in the list now opens the surface that owns its credentials. A postgres one opens its resource in the editor drawer; a Windmill instance one opens the instance modal, which is where its setup checks, password rotation and drop already lived. Both are reachable from the row and from the panel's provenance list, and the provider icon moved inside the button so the whole thing is one target. CustomInstanceDbWizardModal targeted #content unconditionally, which put it underneath the panel drawer that now opens it. It takes a target, and the panel portals it to the body. The status column gains a third state. The probe reports privileges but nothing gated the dot on them, so a data table whose role cannot create tables showed as Connected and only failed when someone ran a migration. It reads "Limited permissions" instead, and opens the panel on the report carrying the GRANTs that fix it -- the settings page has already probed, so the panel takes that report rather than asking the user to run Test connection over work already done. fullyPrivileged is exported from the report component so the dot and the report cannot disagree about what counts as healthy. Co-Authored-By: Claude Opus 5 (1M context) * revert(frontend): keep the data tables settings table as it was The settings table and the setup wizard are two changes that only shared a file. Splitting them makes each reviewable: this branch keeps the wizard, and the read-only row, gear panel, health probe and clickable databases move to their own branch. The rows go back to the editable form with its pickers and save footer, still opening the wizard from Add a database. DataTableSettingsPanel, dataTableHealth and dataTableOrigin had no other consumers and go with them; the connection report stays, because the wizard shows it too. DataTableSettingsType keeps `origin`: the wizard writes it, and the review step reads it back to warn when two data tables would share one database and therefore one _wm_migrations table. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): confirm before dismissing the data table wizard mid-setup Closing was guarded while a run was in flight and unguarded before one, which is backwards: a run leaves a row to resume from, whereas a backdrop click on the review step threw away the project, the pasted password and the folder with nothing to recover them from. Backdrop, Escape and the close button now go through one path that asks first. It only asks when there is something to lose -- no provider chosen yet, or a run that already produced a result, closes immediately -- so the dialog does not become something to click through. Continue in the background still leaves in one click; that exit was always the deliberate one. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): stop the wizard claiming the resource folder controls who can use a data table "Who can use this database" was wrong. Every path that resolves a datatable:// reference -- both executors and the agent-worker endpoint -- reads the resource unchecked, by workspace and name. A resource in u/admin is usable by everyone's scripts. The folder governs who can see and edit the connection, and who can reference the resource directly in a SQL step; neither is who can use the data table. The wizard was contradicting the tab's own description two screens later. The folder select and name field become one Path picker, the same one the resource, variable and script forms use, so the review step reads as a resource path rather than a permission choice. Its initialPath is snapshotted when the step opens: Path seeds itself from it, and a live value fights the typing. Finish now also gates on Path's error, so a taken or malformed path stops the run before it writes anything. The button that opens all this says "Add a data table" -- the data table is what you get; the database is a detail chosen along the way. Co-Authored-By: Claude Opus 5 (1M context) * revert(frontend): move the destructive button restyle out of the wizard PR This reverts 3881e4d8ea. Making default and subtle destructive buttons red at rest changes every existing caller of the prop -- the workspace integrations, AI skills, workspace creation and the instance database drop -- so it is a design-system change, and the call sites it fixed are the migrations list and the database manager. None of that is the setup wizard. Nothing on this branch passes destructive any more, so it leaves with no loose ends. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): make the wizard stepper navigate the steps it already offers Stepper dispatches a click and paints cursor-pointer on every reached step, but the wizard never listened, so the breadcrumbs invited a click and did nothing. They now reach any step already passed, in either direction: going back to check something should not cost the progress, which means tracking the furthest step reached rather than the current one. Forward movement still only happens through the primary action, so a step is never reachable without having been validated -- and changing the intent revokes the steps ahead of it, or Finish could run against a review built from something the user has since edited. The five places that cleared the probe on an edit now do both through one call. During a run nothing is reachable, and the stepper says so rather than showing a pointer over steps that will not respond. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): restore the data tables description lost in the branch split The rewritten description went into DataTableSettings.svelte shortly before that file was restored wholesale to its pre-rebuild state, so it left with the row rework it had nothing to do with. The tab went back to describing the plumbing -- a fully managed PostgreSQL database, reachable from the SDK -- which never answered the question a new user actually has: why this rather than a Postgres resource. It leads with what a data table is, then the two things a resource cannot do -- nobody needs the credentials to query it, and the name can be pointed at another database without editing anything that uses it -- and closes with what Windmill runs on top. Both middle claims are the ones every resolution path backs up: datatable:// resolves by workspace and name, unchecked. Co-Authored-By: Claude Opus 5 (1M context) * fix(backend): say what is missing when a $res: or $var: reference does not resolve Both interpolations fetched with fetch_one and mapped the error through to_anyhow, so a reference to something deleted surfaced as "no rows returned by a query that expected to return at least one row @workspaces.rs:2169". It names neither the kind of thing that was missing nor its path, and it is what a data table pointing at a deleted resource reports. They now fetch_optional and return NotFound naming the path, and datatable resolution adds the data table on the way out: the caller asked for one by name, and a bare "resource f/x/y does not exist" leaves them to work out which of them points at it. The health probe is new, so this string had only just become something users read. Co-Authored-By: Claude Opus 5 (1M context) * feat(frontend): gate the data table wizard behind a dev flag The wizard only appears with `dataTableWizard` set in localStorage; without it the settings page keeps the inline-row flow it had before this branch, down to the empty-state copy and the "New Data Table" button, and the wizard component is not mounted at all. The existing e2e suite drives that button, so the default-off flag is also what keeps it green. Step 2 of "your own database" becomes one list rather than a segmented control: the workspace's Postgres resources, then a New resource card that expands in place. A connection string is not an alternative to a resource, it is how one is written, and the old layout taught otherwise. The card holds the same connection as a string or as fields and carries values across when you switch, so `parse` and `compose` have to be inverses -- hence the percent-encoding on both sides, which also fixes a password containing `@` silently corrupting in the resource form. The Supabase step now uses the same shape. Names and paths are checked as they are typed rather than at the end of a run that may have created a billed project first: the data table name against the charset `edit_datatable_config` enforces, the instance database name against what `setup_custom_instance_db` will accept, and the resource path against both the resource and variable namespaces, since the run writes to both and both writes upsert. `test_datatable_connection_value` refuses `$var:`/`$res:` in its body. It feeds `transform_json_value_unchecked`, which resolves references with no permission check of its own, so an admin could otherwise have had the API server decrypt any workspace secret and hand it to a host the same request chose -- without the audit trail a variable read leaves. Callers testing something unsaved hold the literal value already. Alert, SetupChecklist and postgresConnectionString change for everyone, not just behind the flag: body-only alerts no longer reserve an empty title row, the checklist can nest the checks a step is made of, and the connection-string parser is shared with the resource form. Co-Authored-By: Claude Opus 5 (1M context) * chore: pin ee-repo-ref to the EE branch merged with EE main The Supabase proxies the wizard calls are still unmerged, so the ref cannot be an EE main commit yet; it now names that branch merged with EE main rather than the branch alone, which was nine commits behind and would have been built against a CE main it never saw. Co-Authored-By: Claude Opus 5 (1M context) * feat(frontend): gate the supabase resource path behind the dev flag * test(frontend): pin connection string parsing to libpq behaviour * fix(frontend): keep the supabase resource link off the popup callback path * refactor(frontend): load the supabase resource dialog only behind the flag * fix(frontend): refuse a resource path the wizard run does not own * fix(frontend): let a failed data table setup be corrected without losing what it made * fix(frontend): let a failed setup reuse the resource path it claimed Co-Authored-By: Claude Opus 5 (1M context) * fix(backend): record the two data table connection tests in the audit log Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): use Section for the data table wizard advanced group Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): read connection strings the way libpq does Co-Authored-By: Claude Opus 5 (1M context) * fix(backend): pin the ee ref back to a commit this branch can build Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): keep a failed setup's claims across the redirect and rollback Co-Authored-By: Claude Opus 5 (1M context) * fix(backend): probe a data table with the auth mode the worker will use Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): keep every part of a connection string through the round trip Co-Authored-By: Claude Opus 5 (1M context) * refactor(frontend): give a setup run one record of what it created Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): mark a resource claim by edited_at, not its creator Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): mark every claim by revision, and keep an unconfirmed project's secret Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): refuse to test or save behind a connection string that will not parse Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): refuse a connection string carrying options the resource cannot hold Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): allowlist the connection-string parameters a resource can honour Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): guard every created Supabase project, not just the last one Co-Authored-By: Claude Opus 5 (1M context) * feat: three-step wizard for importing a hub project Importing used to be a single page that inherited whatever workspace happened to be active, with no way to say where the project should go — the hub cannot know, since it only ever links to *an* instance. `/projects/import` now asks: which kind of destination, which workspace, then imports. Nothing is created, switched or written until the last step runs. The wizard's state is a plain value in the URL (`importWizard/plan.ts`), so the back button, the stepper and the Back control are the same operation, and none of them can strand a half-created workspace — there is no state anywhere else to unwind. `importWizard/execution.svelte.ts` is the only code that acts on a plan: it runs create → fetch → import as an observable task list, reuses what already succeeded when retried, and offers to delete the workspace it created if the run stops early. Its UI needs — the data table migration review — are injected, so it holds no components. The old `/projects/install` becomes a redirect: hubs upgrade on their own schedule and a self-hosted one may keep pointing at it for a long time. Co-Authored-By: Claude Opus 5 (1M context) * fix: let the import wizard survive sign-in and a missing workspace Signing in with `rd=/projects/import?hub=...` dropped the destination: the login redirect only honours `rd` verbatim for `/user/workspaces`, so anyone with more than one workspace landed on the workspace picker instead — the page the wizard exists to replace, asking the question it was about to ask. Both copies of that logic now allow the wizard through. The root layout's "no workspace selected" redirect skips the wizard too. It picks the destination itself and may end in a workspace that does not exist yet, so bouncing it to the picker forces the very choice it is there to make. Co-Authored-By: Claude Opus 5 (1M context) * chore: bench page for the import project card /kitchen_sink/import_project_card renders the card against fixtures — a real project, an oversized one, a minimal one — so its layout can be judged without a hub running or an import in flight. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): do not warn about renaming an item that does not exist yet Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): make the review step read as one list of what will exist Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): keep the picked Supabase project across the redirect, reject connect_timeout Co-Authored-By: Claude Opus 5 (1M context) * fix: check the data table connection from a worker, not the API server The wizard's connection check ran on the API server through two endpoints added for it. That server is a different machine with a different identity, so the answer was about the API server rather than about the worker that will run the queries: a host reachable from one is not necessarily reachable from the other, and IAM RDS and Azure workload identity authenticate as whichever process opens the connection. Run the privilege query as a preview job instead. A job goes through the worker's Postgres executor, which is where `PgAuthMode::of` already picks the authentication mode, and it takes either a resource value or a `$res:` path exactly as a Postgres step does. Postgres composes the suggested GRANT statements through `format('%I')`, so identifier quoting stays where it is already implemented. Removes `test_datatable_resource_connection` and `test_datatable_connection_value`, and `connect_as_the_worker_would` with them. Co-Authored-By: Claude Opus 5 (1M context) * refactor: fold check_datatable_connection back into its only caller The helper was split out so the two connection-test endpoints could share a body. Those endpoints are gone, leaving one caller. Co-Authored-By: Claude Opus 5 (1M context) * revert: keep the data table connection check schema inline It was lifted into components so three endpoints could share it. Two of those are gone, so it is back to one user and the extraction changes nothing. Co-Authored-By: Claude Opus 5 (1M context) * fix: restore openapi.yaml to the branch point The previous commit restored main's tip rather than the merge base, which carried three unrelated main-only changes into this branch: the resource mcp_tools truncation fields, the execution_mode description, and a version bump. Co-Authored-By: Claude Opus 5 (1M context) * refactor(frontend): drop four effects from the data table wizard Each was doing work a derived, a load callback or a real entry point does better. - The name conflict is kept with the name it was raised for and derived from it. As an effect it was correct only because it never read what it wrote: the pre-flight sets the message and the effect does not re-trigger, so adding a read would have cleared it the instant it appeared. The message now also comes back if the taken name is retyped, which is what the server will say. - The default resource selection is seeded inside the fetcher that loads the list, where "has the fetch settled" cannot be asked wrong. - Reset-on-open becomes an exported open(), called by the settings page, so a fresh run is set up by the act of opening rather than by a flag emulating mount. - The OAuth connects and the folder list become resources; supabaseAvailable and folders are derived from them. defaultFolder takes the list rather than reading it, so the fetch can seed off its own result. Leaves the debounced path check, which is async with an out-of-order guard. Co-Authored-By: Claude Opus 5 (1M context) * refactor(frontend): drop three effects from the Supabase branch - useSupabaseOauth reports success as onAuthed, alongside the failures it already reported. SupabaseResourceConnect was watching `authed` to find out; it takes the callback instead, keeping the guard that stops an authorization started elsewhere on the page from opening its dialog. - SupabaseProjectStep loads its orgs and projects through a resource keyed on the token, so the `loaded` latch goes and re-authorizing reloads rather than keeping the lists from the expired session. - SetupChecklist records what the user toggled and derives the open state from it, a failed step defaulting to open. Recording the open state instead needed an effect to force it, and that effect re-ran on every progress update, so a description closed while anything was still ticking reopened. A close now holds for the life of the checklist, including across Try again. Leaves the message listener, which subscribes to another window. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): confine the modal restyle to the wizard, and trim the comments The wider side padding and lighter dialog heading were changing all 17 Modal2 dialogs to suit this one flow. They move behind an opt-in `formStyling`, taken by the three dialogs this branch owns; every other Modal2 renders as it did. Also drops two comments that cited a design approval rather than a constraint, and shortens the blocks that had grown past the four lines AGENTS.md asks for. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): use the accent token for the wizard's links `text-blue-500` is the marketing blue `#3B82F6`, which brand-guidelines.md rules out in the app interface. Co-Authored-By: Claude Opus 5 (1M context) * chore: point ee-repo-ref at the EE branch head Picks up EE main, which the branch now needs, and the Supabase proxy auth fix. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): read sslmode by name, and stop decrypting a secret to date it - `sslmode` was found by searching the query text, so it also matched inside another parameter's value: `?application_name=sslmode=disable` passed the allowlist on the parameter name and then parsed as a request to turn TLS off, which both the wizard and the resource form saved and probed. Parsed with `URLSearchParams` by exact name, with a test. - `secretMark` read the variable with `decryptSecret` defaulted to true, so every write decrypted a secret nothing reads and recorded the decryption -- including someone else's on the retry about to refuse it. It wants only `edited_at`, which is returned either way. - The probe gave up at 15s while the worker allows its Postgres connect 20s, so a host that accepts the connection and never answers was cancelled and reported as a missing worker rather than a failed connection. - The create-mode region and project name did not report an intent change, so renaming a project after a name collision left the failure naming the old one. - Two comments described the code as it was before the claim mark became a revision, and a doc comment outlived the field it documented. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): read connection parameters the way libpq does One reader for both the parser and the allowlist, since they disagreed about what a string says in two ways that both ended in a weaker connection than was pasted: - `URLSearchParams.get` takes the first of a repeated parameter and libpq takes the last, so `?sslmode=disable&sslmode=require` was read as `disable`. - The allowlist folded the parameter name and the parser did not, so `?SslMode=verify-full` was refused by neither and honoured by neither, and saved as the `require` default. The parked Supabase run is now handed to `open()` rather than read back off the `resume` prop it was just assigned to, so restoring it does not depend on when that prop reaches the component. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): keep connection parameter names case-sensitive libpq does not fold them: `?SslMode=disable` is rejected as an invalid URI query parameter rather than read as `sslmode`, which a local server confirms. Folding made Windmill accept and honour a string Postgres itself refuses; naming the parameter instead tells the user why it cannot be stored. The last-value-wins rule for a repeated parameter is unchanged, and matches what the same server does. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): seed the Supabase organization from the project it selects The loader took `orgs[0]` independently of the project it seeded, so an account whose first project sits outside its first organization had the review step name an organization the database does not belong to. Picking a project by hand already derives it; the seeding now does the same. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): let the probe report an empty search_path instead of failing on it `format('%I', NULL)` raises rather than returning NULL, so a role whose search_path names no valid schema failed the whole privilege query and was reported as an unreachable database. That is the one case `fix_search_path` exists to name, and it never reached the user. Verified against a local server with `SET search_path = ''`. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): say which of the two refusals a connection string hit Making parameter names case-sensitive gave `unsupportedConnectionParam` two reasons to refuse, and the single message explained only one. `?SslMode=` was answered with "Windmill cannot store SslMode on a Postgres resource", which is false twice over: sslmode is exactly what the resource stores, and the string asks for nothing because Postgres rejects the URI. It now names the spelling when the parameter is one we keep, and the storage limit otherwise. The folder-list guard also still read the `resume` prop that `open(parked)` was changed to stop trusting, so the resumed path now comes from whatever `reset` was handed. Co-Authored-By: Claude Opus 5 (1M context) * fix(frontend): leave the Supabase organization unset when the lookup misses Falling back to the first organization named one the seeded project is not in, since `supabaseSummary` prefers `intent.org` over the project's own. Unset, it falls through to the project's organization identifier — the right one, spelled as a slug rather than a name. Co-Authored-By: Claude Opus 5 (1M context) * fix: harden the import wizard and put it on the design system Review fixes, then the parts of the wizard that were hand-built where the design system already had an answer. Correctness: - Hub SVGs are sanitised with DOMPurify before `{@html}`. The earlier comment claimed the markup came from the hub's own icon package rather than user input, which the custom-URL feature makes false: the hub is whatever address the user typed. - The run owns navigation while it is in flight. The stepper refuses to move, `beforeNavigate` cancels browser back/forward, and unmounting resolves a pending migration review so the executor cannot hang waiting on a component that is gone. - The folder edited on the last step reaches the executor, so a retry after changing it imports where the field now says. - `validateWorkspaceId` and the workspace-entry pair (`listUserWorkspaces` then `switchWorkspace`) are extracted, so the wizard and the real create form cannot drift on what an id is or on what entering a workspace means. Design system: - The destination tiles are `RadioCard`, which gains `showRadio` and a snippet `description`; the wizard turns the glyph off because the border and tint already say which one is picked. `RadioCard` now also carries `role="radio"` and `aria-checked`, which it had neither of, and marks its selection with `surface-accent-selected` — the token `FileExplorer`, `TriggersTable` and `RunnableRow` all use for the chosen row. - Form labels follow `brand-guidelines.md` — sentence case, real `
    {#if step.substeps?.length} -
    - +
    +
    {/if}
    diff --git a/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte b/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte index 2e19cbea82..3495bdbee0 100644 --- a/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte +++ b/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte @@ -21,6 +21,8 @@ FolderService, OauthService, ResourceService, + UserService, + type User, VariableService, WorkspaceService } from '$lib/gen' @@ -85,6 +87,38 @@ customInstanceDbs: ResourceReturn confirmationModal: ConfirmationModalHandle defaultInstanceDbName: () => string + /** Name to open with, when the caller needs a table of a particular name rather + * than whatever the user picks — the import wizard configures the one a project's + * migrations target. + * + * Locked when `onFinishAlso` is also given, because that work targets this name and + * nothing carries an edit through to it: renaming `main` to `other` would create + * `other`, then run the migrations against `main`, fail, and leave a data table + * nobody asked for. Editable without one, where the name is only a name. */ + initialName?: string + /** Where the dialog portals to. `#content` is the app shell's scroll container, + * which only exists inside the `(logged)` layout; a page reparented out of it + * (the hub import wizard) has to say `body` or the portal finds nothing and the + * dialog never appears. */ + modalTarget?: string + /** What the caller does once the table exists, named on the final button so the + * user is told before pressing it — the import wizard runs the project's + * migrations, which is otherwise invisible until it has already happened. */ + finishAlso?: string + /** The work `finishAlso` names. Run as the last checklist step, so it reports + * where the rest of the run does instead of starting after the dialog closes. + * Throwing marks that step failed; the table itself is already made either way. */ + onFinishAlso?: () => Promise + /** The workspace everything here is created in and checked against. + * + * Defaults to `$workspaceStore`, which is right for the settings page — it is the + * workspace being looked at. The import wizard is the exception: its page is + * reparented out of `(logged)`, so nothing re-runs the layout's workspace + * persistence, and after a reload the store still names whatever workspace the + * user came from while the plan in the URL names the destination. Left ambient, + * this would create the data table in one workspace and run the project's + * migrations in the other. */ + workspace?: string } let { @@ -95,9 +129,62 @@ onDone, customInstanceDbs, confirmationModal, - defaultInstanceDbName + defaultInstanceDbName, + initialName, + modalTarget = '#content', + finishAlso, + onFinishAlso, + workspace: workspaceProp }: Props = $props() + /** + * The caller needs this exact table, and has follow-up work bound to its name. + * + * Captured when the dialog opens rather than read live: `initialName` is the caller's + * `wizardFor`, which it clears from `onDone` — and that fires after a *failed* run too, + * while the dialog stays up offering Back. Reading it live releases the lock exactly when + * the user is most likely to edit, which is the divergence the lock exists to stop. + */ + let nameLocked = $state(false) + + /** Every write and every check goes through this, never `$workspaceStore` directly. */ + const targetWorkspace = $derived(workspaceProp ?? $workspaceStore ?? '') + + /** + * Who the caller is *in the destination*, which is not who `$userStore` describes. + * + * `$userStore` is the membership of the workspace the app is in. Routing the API calls + * elsewhere without routing this leaves the username behind: after a reload on the import + * wizard's step 4 it names the workspace the user came from, and a resource path built + * from it lands on `u/` inside the destination — failing an ownership check, + * or for an admin, quietly putting database credentials in another member's namespace. + */ + let targetUser = $state(undefined) + const aimedElsewhere = $derived(!!workspaceProp && workspaceProp !== $workspaceStore) + const ambientUsername = $derived($userStore?.username ?? '') + const targetUsername = $derived(aimedElsewhere ? (targetUser?.username ?? '') : ambientUsername) + /** The destination's membership could not be read, so nothing here knows who the user is. */ + let membershipFailed = $state(false) + + async function loadTargetUser(): Promise { + const ws = workspaceProp + if (!ws || ws === $workspaceStore) { + targetUser = undefined + membershipFailed = false + return + } + try { + targetUser = await UserService.whoami({ workspace: ws }) + membershipFailed = false + } catch { + // Recorded rather than swallowed: an unknown username silently becomes `admin` in + // the default path, which is the wrong namespace to write credentials into. Setup + // is blocked instead. + targetUser = undefined + membershipFailed = true + } + } + const STEPS = ['Choose a database', 'Set it up', 'Review'] let wiz: WizardState = $state( @@ -146,7 +233,7 @@ } clearTimeout(variableCheck) variableCheck = setTimeout(async () => { - const taken = await VariableService.existsVariable({ workspace: $workspaceStore!, path }) + const taken = await VariableService.existsVariable({ workspace: targetWorkspace, path }) // Two checks can be in flight at once and resolve out of order. A `false` for a path // nobody is on any more would clear the error guarding the one about to be written; // a `true` would disable Finish over a path this run stopped caring about. @@ -163,7 +250,7 @@ * in flight when Finish is pressed. */ async function pathConflictMessage(path: string): Promise { - const workspace = $workspaceStore! + const workspace = targetWorkspace // Each namespace answers to its own claim. Holding the secret says nothing about who owns // the resource beside it, so one claim must not wave the other's check through. const [variable, resource] = await Promise.all([ @@ -178,11 +265,14 @@ let maxStep = $state(1) function defaultProjectName(): string { - return `windmill-${$workspaceStore ?? 'workspace'}` + return `windmill-${targetWorkspace || 'workspace'}` } function defaultTableName(): string { - return existingNames.includes('main') ? `${$workspaceStore ?? 'data'}_datatable` : 'main' + // A caller that needs a specific name wins over the usual "main, unless taken": + // the import wizard's migrations only apply to a table of the name they target. + if (initialName) return initialName + return existingNames.includes('main') ? `${targetWorkspace || 'data'}_datatable` : 'main' } // Takes the list rather than reading it, so the fetch that loads it can seed off its own @@ -190,7 +280,7 @@ function defaultFolder(list: string[] = folders): string { // The first folder this admin can write to, so the resource lands somewhere the team // can find and repair. A workspace with no folders falls back to the personal space. - return list.length ? `f/${list[0]}` : `u/${$userStore?.username ?? 'admin'}` + return list.length ? `f/${list[0]}` : `u/${targetUsername || 'admin'}` } // A row this run wrote and could not take back out is still its own: `removeRow` reports @@ -236,7 +326,7 @@ ) const pgResources = resource( - () => (opened && wiz.provider === 'resource' ? ($workspaceStore ?? '') : ''), + () => (opened && wiz.provider === 'resource' ? targetWorkspace : ''), async (workspace) => { if (!workspace) return undefined const list = await ResourceService.listResource({ workspace, resourceType: 'postgresql' }) @@ -330,7 +420,7 @@ ) const folderNames = resource( - () => (opened ? ($workspaceStore ?? '') : ''), + () => (opened ? targetWorkspace : ''), async (workspace) => { if (!workspace) return [] const all = await FolderService.listFolderNames({ workspace }) @@ -374,7 +464,11 @@ let resumedPath = $state(undefined) function reset(from: WizardResume | undefined) { + nameLocked = !!initialName && !!onFinishAlso resumedPath = from?.resourcePath + // A pending confirmation that never settled leaves `dismissing` true, and `finally` + // cannot clear what never resolves — so a fresh open always starts dismissable. + dismissing = false wiz = newWizardState({ name: from?.name || defaultTableName(), projectName: from?.projectName || defaultProjectName(), @@ -391,6 +485,7 @@ createdProjects = [] nameConflictFor = undefined lastFailure = '' + finishAlsoFailed = false pathTakenError = '' poolerUnavailable = undefined if (from) { @@ -424,7 +519,11 @@ * what keeps the restore independent of when the prop it was assigned to reaches this * component. */ - export function open(parked?: WizardResume) { + export async function open(parked?: WizardResume) { + // Awaited before `reset`, which seeds the resource path from `defaultFolder()` and so + // needs the destination's username. Seeding first and correcting later loses whenever + // the folder list resolves first, and never corrects at all if `whoami` fails. + await loadTargetUser() reset(parked ?? resume) opened = true } @@ -472,12 +571,14 @@ // Also retires any check still in flight, so its answer cannot land on the edited value. probeToken++ clearProbe(wiz) - // Read off one attempt against one project; the review step would otherwise warn about - // a limitation that no longer applies while claiming session pooling right above it. + // Read off one attempt against one project, so it does not survive a change of inputs: + // the review step would otherwise warn about a limitation that does not apply to what + // it is describing, while claiming session pooling right above it. poolerUnavailable = undefined // Same for the failure carried back to the review step: it names inputs that have since // been edited, so it would describe a run nobody can still act on. lastFailure = '' + finishAlsoFailed = false if (maxStep > wiz.step) maxStep = wiz.step } @@ -527,7 +628,7 @@ settle({ checking: false, report: undefined, error: undefined }) return } - const report = await probeDatatableConnection($workspaceStore!, database) + const report = await probeDatatableConnection(targetWorkspace, database) settle({ checking: false, report, error: undefined }) } catch (err: any) { settle({ @@ -579,6 +680,13 @@ ) /** Why the last run failed, kept on the review step after the checklist is dropped. */ let lastFailure = $state('') + /** + * The appended `onFinishAlso` step failed while `runSetup` itself succeeded. Tracked apart + * from `run.result`, which stays the setup's own verdict: the data table really was + * created, so a retry must re-run only this last step. Re-running the setup would ask for + * the table name it has just taken, and be refused as a duplicate. + */ + let finishAlsoFailed = $state(false) /** * A refused pre-flight means nothing ran, so the checklist from a previous attempt has to @@ -631,7 +739,7 @@ const name = wiz.review.name.trim() try { if (claimedName !== name) { - const settings = await WorkspaceService.getSettings({ workspace: $workspaceStore! }) + const settings = await WorkspaceService.getSettings({ workspace: targetWorkspace }) if (settings.datatable?.datatables?.[name]) { nameConflictFor = { name, @@ -666,6 +774,7 @@ } run = { steps: planSteps(wiz), running: true } lastFailure = '' + finishAlsoFailed = false // The database is registered by the call whatever it answers, so asking for one is // already leaving something behind. if (wiz.provider === 'instance' && wiz.instance.mode === 'create') { @@ -675,7 +784,7 @@ let result: RunResult | undefined = undefined try { result = await runSetup(wiz, { - workspace: $workspaceStore!, + workspace: targetWorkspace, supabaseToken: supaOauth.token, onInstanceDbsChanged: async () => { await customInstanceDbs.refetch() @@ -684,9 +793,27 @@ onPoolerUnavailable: (reason) => (poolerUnavailable = reason), createdProjects, claims, - username: $userStore?.username ?? '' + username: targetUsername }) } finally { + // The caller's own finishing work, appended to the same checklist. It only runs + // on a clean setup: there is no table for it to act on otherwise. + if (result?.ok && onFinishAlso && finishAlso) { + const title = finishAlso.charAt(0).toUpperCase() + finishAlso.slice(1) + run.steps = [...run.steps, { title, status: 'running' }] + try { + await onFinishAlso() + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'done' as const } : s + ) + } catch (err: any) { + const description = err?.body ?? err?.message ?? String(err) + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'failed' as const, description } : s + ) + finishAlsoFailed = true + } + } // `runSetup` catches per step, but anything escaping it would otherwise leave the // button spinning with a page reload the only way out. // Kept, not replaced: what an earlier attempt wrote is still out there, so a later @@ -713,7 +840,10 @@ /** * Whether closing would throw away work. A failed run counts: its inputs are still editable * and it may have left something behind. A run in flight cannot be closed at all, and one - * that succeeded has nothing left to lose. + * that made its data table has nothing left to lose — including when `onFinishAlso` failed + * afterwards, because the table is real and working and the caller owns what is left. The + * import step, the only caller that passes one, shows that failure on its own row with a + * way to run it again and will not let Finish through while it stands. */ function hasUnfinishedIntent(): boolean { return wiz.provider !== undefined && !run.running && !run.result?.ok @@ -730,19 +860,52 @@ return } dismissing = true - const confirmed = await confirmationModal.ask({ - title: 'Leave without adding a data table?', - // A run that failed and was sent back to be edited leaves whatever it got through - // behind it, so promising otherwise would be a lie exactly when it matters most. - children: leftBehind - ? 'The setup that already ran left what it created behind, and what you have filled in here will be lost.' - : 'Nothing has been created yet, and what you have filled in here will be lost.', - confirmationText: 'Discard' - }) - dismissing = false - // Re-read rather than trust the entry check: a run can start while the dialog is up, and - // answering Discard would otherwise tear the modal down in the middle of it. - if (confirmed && !preventClose) close() + // `finally`, because the flag is what blocks a second attempt: an `ask` that throws + // would otherwise leave the dialog permanently undismissable — the backdrop, Escape + // and the close button all return early here, so the only way out would be a reload. + try { + const confirmed = await confirmationModal.ask({ + title: 'Leave without adding a data table?', + // A run that failed and was sent back to be edited leaves whatever it got through + // behind it, so promising otherwise would be a lie exactly when it matters most. + children: leftBehind + ? 'The setup that already ran left what it created behind, and what you have filled in here will be lost.' + : 'Nothing has been created yet, and what you have filled in here will be lost.', + confirmationText: 'Discard' + }) + // Re-read rather than trust the entry check: a run can start while the dialog is up, and + // answering Discard would otherwise tear the modal down in the middle of it. + if (confirmed && !preventClose) close() + } finally { + dismissing = false + } + } + + /** Re-runs only the appended step, which is the only thing that failed. */ + async function retryFinishAlso() { + if (!onFinishAlso || !finishAlso) return + const title = finishAlso.charAt(0).toUpperCase() + finishAlso.slice(1) + finishAlsoFailed = false + run = { + ...run, + running: true, + steps: [...run.steps.slice(0, -1), { title, status: 'running' as const }] + } + try { + await onFinishAlso() + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'done' as const } : s + ) + } catch (err: any) { + const description = err?.body ?? err?.message ?? String(err) + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'failed' as const, description } : s + ) + finishAlsoFailed = true + } finally { + run = { ...run, running: false } + onDone() + } } function close() { @@ -752,10 +915,18 @@ // The single primary action. Its label says what it is about to do, and doing it is what // moves the wizard on. let primary = $derived.by(() => { + // Ahead of everything: without the destination's membership the resource path would be + // guessed, and a guess here writes database credentials into somebody else's namespace. + if (aimedElsewhere && membershipFailed) + return { label: 'Cannot read your access to this workspace', disabled: true } if (submitting && !run.running) return { label: 'Setting things up', disabled: true, busy: true } if (run.steps.length) { if (run.running) return { label: 'Setting things up', disabled: true, busy: true } + // Before the `ok` check: the setup succeeded and the step after it did not, so + // "Done" would be offered over a failed row. + if (finishAlsoFailed) + return { label: 'Try again', disabled: false, act: retryFinishAlso } if (run.result?.ok) return { label: 'Done', disabled: false, act: close } // A run that died because the Supabase token expired would retry into the same 401 // forever; authorizing again is the only thing that can move it on. @@ -809,11 +980,12 @@ act: enterReview } } + const created = + wiz.provider === 'supabase' && wiz.supabase.mode === 'create' + ? 'Create project and data table' + : 'Create data table' return { - label: - wiz.provider === 'supabase' && wiz.supabase.mode === 'create' - ? 'Create project and data table' - : 'Create data table', + label: finishAlso ? `${created} and ${finishAlso}` : created, disabled: // Guards the way back as well as the way forward: the stepper can return to step 2, // and not every control there invalidates the review it just made stale. @@ -842,7 +1014,7 @@ else opened = v } } - target="#content" + target={modalTarget} formStyling title="Add a data table" contentClasses="flex flex-col" @@ -1034,7 +1206,7 @@ {#if wiz.instance.mode === 'existing'} {@const shared = ( customInstanceDbs.current?.[wiz.instance.dbName ?? '']?.used_by_workspaces ?? [] - ).filter((w) => w !== $workspaceStore)} + ).filter((w) => w !== targetWorkspace)} {#if shared.length} @@ -1047,7 +1219,7 @@
    {#each instanceDbs as { name, db } (name)} {@const selected = wiz.instance.dbName === name} - {@const others = (db.used_by_workspaces ?? []).filter((w) => w !== $workspaceStore)} + {@const others = (db.used_by_workspaces ?? []).filter((w) => w !== targetWorkspace)} +
    +
    +
    + {:else if step === 2} +
    + {#if !choiceIsExisting} +
    +

    Name the new workspace

    +
    + +
    + + +
    + + {#if !automateUsername} + + {/if} + {:else} +
    +

    Pick a workspace

    +

    The project is imported into this one.

    +
    + + {#if workspaceList.loading} +
    + Loading your workspaces… +
    + {:else if workspaceList.error} +

    + Could not list your workspaces. Reload the page, or go back and create a new one. +

    + {:else if workspaces.length === 0} +

    + You are not a member of any workspace yet. Go back and create one, or ask an admin to + invite you. +

    + {:else} + + {#if workspaces.length > 1} +
    +
    + + +
    + {#if hasForks} + + {/if} +
    + {/if} + + + + {/if} + {/if} + +
    + + {#if !choiceIsExisting} + + {/if} +
    +
    + {:else if step === 3} + go({ folder }, 3, { replace: true })} + onFinish={() => + setupNeeded + ? // Replaces rather than pushes: after a reload on step 4 the run is gone, and + // a step-3 entry in history is a browser-Back route to the same fresh import + // the stepper is now blocked from reaching. + go({}, 4, { replace: true }) + : finish()} + onBack={() => go({}, 2)} + onExecution={(e) => (execution = e)} + resume={execution} + /> + {:else} + go({}, 3) : undefined} + /> + {/if} + +{/if} diff --git a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte b/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte deleted file mode 100644 index 06f2344356..0000000000 --- a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte +++ /dev/null @@ -1,374 +0,0 @@ - - -
    - {#if !slug} -

    Missing ?hub=<slug>.

    - {:else if loading} -
    - Loading project… -
    - {:else if loadError} -

    Failed to load project: {loadError}

    - {:else if data} -

    Add “{data.project.name}” to workspace

    -

    {data.project.summary}

    - -
    -

    - Folder in {workspace} -

    - -

    - Items import under f/{folderName.trim() || data.project.slug}/. -

    -
    - -
    - {counts?.scripts} scripts - {counts?.flows} flows - {counts?.apps} apps - {counts?.resources} resources - {counts?.triggers} triggers - {#if counts && counts.migrations > 0} - {counts.migrations} data table migrations - {/if} -
    - -
    - Resources are imported as empty stubs — set their values after import; a resource whose path - already exists is reported as failed (existing values are never overwritten). Trigger kinds - are recreated disabled, except GCP and Azure triggers, which manage cloud subscriptions at - creation and must be re-created manually after filling their resource. Kafka, NATS, SQS, GCP - and Azure triggers all require Enterprise. Triggers that reference a resource depend on stubs - imported empty, so fill in the resource value before re-enabling the trigger. -
    - -
    - - {#if done} - - {/if} -
    - - {#if results.length} -
      - {#each results as r} -
    • - {r.ok ? '✓' : '✗'} - {r.path} - {#if !r.ok}— {r.error}{/if} -
      • - {/each} -
    - {/if} - {/if} -
    - - - - - - closeMigrationReview(false)}> - closeMigrationReview(false)}> -
    -

    - This project ships migrations that recreate the data tables it uses. Review and edit the - SQL, then choose which to run. A migration runs against the data table of the same name in - {workspace}; if that data table has migrations enabled it is - recorded, otherwise it runs once as a preview job. -

    - {#each reviewList as m (m.datatable_name)} -
    -
    - {m.datatable_name} - -
    - {#if m.run} - - {/if} -
    - {/each} -
    - {#snippet actions()} - - - {/snippet} -     -     diff --git a/frontend/src/routes/(root)/(logged)/projects/install/+page.ts b/frontend/src/routes/(root)/(logged)/projects/install/+page.ts new file mode 100644 index 0000000000..718b624bd4 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/projects/install/+page.ts @@ -0,0 +1,13 @@ +import { redirect } from '@sveltejs/kit' +import { base } from '$app/paths' +import type { PageLoad } from './$types' + +/** + * `/projects/install?hub=` was where the hub's "Add to workspace" button + * pointed before the import wizard existed. Hubs upgrade on their own schedule — + * a self-hosted one may keep sending people here for a long time — so the old + * entry point forwards to the wizard rather than 404ing, query string intact. + */ +export const load: PageLoad = ({ url }) => { + redirect(307, `${base}/projects/import${url.search}`) +} diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte index d7520bd873..df85c1f1c1 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte index c4d2168d26..ed9bf2b11e 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte @@ -26,6 +26,7 @@ import { switchWorkspace } from '$lib/storeUtils' import { GitFork, Settings, User, Search, ChevronsDownUp, ChevronsUpDown } from 'lucide-svelte' import { isCloudHosted } from '$lib/cloud' + import { canCreateWorkspace } from '$lib/workspaceCreation' import AnimatedButton from '$lib/components/common/button/AnimatedButton.svelte' import { emptyString } from '$lib/utils' import { getUserExt } from '$lib/user' @@ -104,9 +105,7 @@ let onlyAdminsWorkspace = $derived(allWorkspaces.length === 1 && allWorkspaces[0].id === 'admins') async function getCreateWorkspaceRequireSuperadmin() { - const r = await fetch(base + '/api/workspaces/create_workspace_require_superadmin') - const t = await r.text() - createWorkspace = t != 'true' + createWorkspace = await canCreateWorkspace(false) } let createWorkspace = $state($superadmin || isCloudHosted()) diff --git a/frontend/src/routes/(root)/+layout.svelte b/frontend/src/routes/(root)/+layout.svelte index 1f7a813eac..0e404afbcc 100644 --- a/frontend/src/routes/(root)/+layout.svelte +++ b/frontend/src/routes/(root)/+layout.svelte @@ -1,4 +1,5 @@ @@ -154,17 +145,15 @@ {#snippet actions()} - - - + {/snippet} @@ -260,8 +249,8 @@
    Developers

    - Calculated on the MAXIMUM number of users in a given billing - period, see the Customer Portal for more info. + Calculated on the MAXIMUM number of users in a given billing period, see the + Customer Portal for more info.

    @@ -276,8 +265,8 @@
    Operators

    - Calculated on the MAXIMUM number of operators in a given - billing period, see the Customer Portal for more info. + Calculated on the MAXIMUM number of operators in a given billing period, see + the Customer Portal for more info.

    @@ -295,8 +284,9 @@ 1 developer = 1 seat, 2 operators = 1 seat.

    - u = ceil({formatNumber(premiumInfo.developerNb)} + {formatNumber(premiumInfo.operatorNb)}/2) - = {formatNumber(premiumInfo.seatsFromUsers)} + u = ceil({formatNumber(premiumInfo.developerNb)} + {formatNumber( + premiumInfo.operatorNb + )}/2) = {formatNumber(premiumInfo.seatsFromUsers)}

    @@ -311,9 +301,8 @@
    Executions this month

    - One execution equals one job - up to 1 second on a worker with 2GB of memory, with each additional - second counting as an extra execution. + One execution equals one job up to 1 second on a worker with 2GB of memory, + with each additional second counting as an extra execution.

    @@ -365,8 +354,8 @@ Used seats (billed)

    - Highest between seats from 'Developers + Operators' and 'Seats from executions'. - This is the number of seats used for billing this month. + Highest between seats from 'Developers + Operators' and 'Seats from + executions'. This is the number of seats used for billing this month.

    u + c = {formatNumber(premiumInfo.usedSeats)} @@ -398,8 +387,8 @@

    Estimate your monthly cost

    - This is a rough estimate based on your expected team size and workload. Actual billing is based - on the maximum number of users and executions in a given month. + This is a rough estimate based on your expected team size and workload. Actual billing is + based on the maximum number of users and executions in a given month.

    @@ -420,13 +409,13 @@
    Operators
    - {estimatedOps} operator{estimatedOps === 1 ? '' : 's'} + {estimatedOps} operator{estimatedOps === 1 + ? '' + : 's'}
    -

    - 2 operators = 1 seat -

    +

    2 operators = 1 seat

    @@ -434,8 +423,8 @@
    Monthly executions
    - One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, with - each additional second counting as an extra execution. + One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, + with each additional second counting as an extra execution.
    @@ -449,9 +438,7 @@ format={(v) => `${v * 10}k`} hideInput /> -

    - Each seat includes 10k executions per month. -

    +

    Each seat includes 10k executions per month.

    @@ -558,8 +545,8 @@
  • Every seat includes 10 000 executions - One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, with - each additional second counting as an extra execution. + One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, + with each additional second counting as an extra execution.
    • {:else} @@ -586,9 +573,7 @@ {/if} {:else} -
    - Workspace is on the team plan -
    +
    Workspace is on the team plan
    {/if} {:else if planTitle == 'Enterprise'} {#if plan != 'enterprise'} @@ -601,9 +586,7 @@ See more {:else} -
    - Workspace is on enterprise plan -
    +
    Workspace is on enterprise plan
    {/if} {:else if planTitle === 'Free'} {#if plan} @@ -611,9 +594,7 @@ Cancel your plan in the Customer Portal to downgrade to the free plan {:else} -
    - Workspace is on the free plan -
    +
    Workspace is on the free plan
    {/if} {/if} diff --git a/frontend/src/lib/components/sidebar/SidebarUsage.svelte b/frontend/src/lib/components/sidebar/SidebarUsage.svelte index e62cd02565..1625ed83e2 100644 --- a/frontend/src/lib/components/sidebar/SidebarUsage.svelte +++ b/frontend/src/lib/components/sidebar/SidebarUsage.svelte @@ -2,21 +2,18 @@ import { resource } from 'runed' import { goto } from '$lib/navigation' import { isCloudHosted } from '$lib/cloud' - import { UserService } from '$lib/gen' + import { WorkspaceService } from '$lib/gen' import { isPremiumStore, usageStore, userStore, - userWorkspaces, workspaceMembershipVersion, workspaceStore, - workspaceUsageStore, - type UserWorkspace + workspaceUsageStore } from '$lib/stores' import { refreshExecutions } from '$lib/usage.svelte' import { logFeatureUsage } from '$lib/utils/featureUsage' import { scopedValue, tagged } from '$lib/utils/scopedValue' - import { findWorkspaceAncestors } from '$lib/utils/workspaceHierarchy' import { Button } from '$lib/components/common' import Modal from '$lib/components/common/modal/Modal.svelte' import { Tooltip } from '$lib/components/meltComponents' @@ -30,51 +27,30 @@ let open = $state(false) - // A fork's usage and tier resolve to its billing root while its member list is a - // subset of the root's, so seats must come from the root or the cap is fork-sized - // against root usage. `undefined` when the root isn't visible from here: the cap - // is then unknowable, and the caller hides the meter rather than guessing. - function billingRoot(workspace: string, all: UserWorkspace[]): string | undefined { - const self = all.find((w) => w.id === workspace) - if (!self) return undefined - if (!self.parent_workspace_id) return workspace - const top = findWorkspaceAncestors(workspace, all).at(-1) - return top && !top.parent_workspace_id ? top.id : undefined - } + // Seat count for a paid workspace, the basis of its included executions. The server + // resolves a fork to the workspace its plan is billed on and counts the seats there, + // because neither is answerable from here: a fork's member list is a subset of that + // root's, and a fork member need not be a member of the root at all. + const fetchSeats = tagged( + async (workspace: string) => (await WorkspaceService.getBillableSeats({ workspace })).seats + ) - // Seat count for a paid workspace, the basis of its included executions. Only the - // user list is needed: `premium_info` carries the same usage number as - // `workspaceUsageStore` but requires admin and only exists when Stripe is - // configured, so it would leave regular members with no block at all. - const fetchSeats = tagged(async (root: string) => { - // Throws for a fork member with no seat in the root, which is the same answer as - // an unresolvable root: leave the paid meter hidden. - const users = await UserService.listUsers({ workspace: root }) - // Same basis as the backend's `count_paid_seats`: disabled members and service - // accounts are not billed, so counting them inflates the cap and hides a real - // overage. 1 developer = 1 seat, 2 operators = 1 seat. - const billable = users.filter((u) => !u.disabled && !u.is_service_account) - const developers = billable.filter((u) => !u.operator).length - const operators = billable.length - developers - return Math.ceil(developers + operators / 2) - }) - - const billingRootId = $derived.by(() => { - const workspace = $workspaceStore - if (!isCloudHosted() || !$isPremiumStore || !workspace) return undefined - return billingRoot(workspace, $userWorkspaces ?? []) - }) + const meteredWorkspace = $derived( + isCloudHosted() && $isPremiumStore ? $workspaceStore : undefined + ) // The membership version is in the key so a change re-resolves the cap, but not in // the tag: tagging by it would blank the bar on every change. const seatsResource = resource( () => - billingRootId ? { root: billingRootId, version: $workspaceMembershipVersion } : undefined, - async (key) => (key ? await fetchSeats(key.root) : undefined) + meteredWorkspace + ? { workspace: meteredWorkspace, version: $workspaceMembershipVersion } + : undefined, + async (key) => (key ? await fetchSeats(key.workspace) : undefined) ) const scopedSeats = scopedValue() - const seats = $derived(scopedSeats(billingRootId, seatsResource.current)) + const seats = $derived(scopedSeats(meteredWorkspace, seatsResource.current)) type QuotaKey = 'user' | 'workspace' diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 782da1bb2c..8dd2937273 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -41,7 +41,7 @@ import { sendUserToast } from '$lib/toast' import { clone, emptyString, encodeState, hasUnsavedChanges } from '$lib/utils' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' - import { Slack, Target } from 'lucide-svelte' + import { ExternalLink, Slack, Target } from 'lucide-svelte' import SidebarNavigation from '$lib/components/common/sidebar/SidebarNavigation.svelte' import PremiumInfo from '$lib/components/settings/PremiumInfo.svelte' @@ -1472,9 +1472,30 @@ This workspace is a fork of {currentWorkspace.parent_workspace_id}. It runs on the parent's plan and its executions count toward the parent's usage and bill, - so there is no separate subscription here. Manage billing, seats, and quotas from - the parent workspace's settings. + so it is never invoiced separately. Manage billing, seats, and quotas from the + parent workspace's settings. + {#if plan} +
    + + It is on a paid plan that is billed on its own, so this workspace is paid for + twice. Cancel that subscription in the customer portal to keep only + {currentWorkspace.parent_workspace_id}'s plan. This workspace keeps + running either way, on the parent's plan. + {#if customer_id} +
    + +
    + {/if} +     +
    + {/if} {:else} {/if} From 7a0c81d7222f3e7bb971c3cb9baeb82f153ba749 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 28 Aug 2026 17:26:18 +0200 Subject: [PATCH 22/40] chore(main): release 1.799.0 (#10874) * chore(main): release 1.799.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 18 + backend/Cargo.lock | 377 +++++++++++------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 +-- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 48 ++- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- windmill-yaml-validator/package-lock.json | 4 +- windmill-yaml-validator/package.json | 2 +- 20 files changed, 333 insertions(+), 194 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 2c5c3efad9..c46e4845e3 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.798.1" + ".": "1.799.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index c4b042bab5..aa9bd16eac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [1.799.0](https://github.com/windmill-labs/windmill/compare/v1.798.1...v1.799.0) (2026-08-28) + + +### Features + +* enable Anthropic prompt caching on Vertex AI agent steps ([#10876](https://github.com/windmill-labs/windmill/issues/10876)) ([320f400](https://github.com/windmill-labs/windmill/commit/320f4005124202852e6e9c70b394e7f87231d278)) +* instrument AI fill/fix, evals, agents and the debugger ([#10853](https://github.com/windmill-labs/windmill/issues/10853)) ([0bbd559](https://github.com/windmill-labs/windmill/commit/0bbd559ac8a35dba04ba5e8d6f2fd8d1d1124891)) + + +### Bug Fixes + +* **datatables:** stop a fork's pg_dump restore from failing silently ([#10830](https://github.com/windmill-labs/windmill/issues/10830)) ([3ce9bbc](https://github.com/windmill-labs/windmill/commit/3ce9bbc7168b837cb2111aabd533bb67803502b8)) +* key build artifact caches on a runnable's inline modules ([#10819](https://github.com/windmill-labs/windmill/issues/10819)) ([b72ccc3](https://github.com/windmill-labs/windmill/commit/b72ccc35934165b4bad112b947ca5af064aab26f)) +* nested template literals in step inputs, and unresolvable $args tags ([#10856](https://github.com/windmill-labs/windmill/issues/10856)) ([8f349c0](https://github.com/windmill-labs/windmill/commit/8f349c032a0d75fc3350292075e5050a030f6166)) +* pre-fill the test panel JSON args editor and align its placeholder ([#10871](https://github.com/windmill-labs/windmill/issues/10871)) ([fb82f36](https://github.com/windmill-labs/windmill/commit/fb82f36e6d6492dd0740984d8d78ea4eaa30361e)) +* reject a prefixed error_handler_path on triggers ([#10847](https://github.com/windmill-labs/windmill/issues/10847)) ([d334831](https://github.com/windmill-labs/windmill/commit/d33483173526a3b352d2829ac8a2e1e229cc1127)) +* unify billable seat counting and prevent fork subscriptions ([#10818](https://github.com/windmill-labs/windmill/issues/10818)) ([7dd88c4](https://github.com/windmill-labs/windmill/commit/7dd88c470caee5f095dc240667aa7550c55696bc)) + ## [1.798.1](https://github.com/windmill-labs/windmill/compare/v1.798.0...v1.798.1) (2026-08-27) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 0701814b4b..b7f278d437 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -23,7 +23,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -261,13 +261,13 @@ dependencies = [ [[package]] name = "argon2" -version = "0.5.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c" dependencies = [ "base64ct", - "blake2", - "cpufeatures 0.2.17", + "blake2 0.11.0", + "cpufeatures 0.3.1", "password-hash", ] @@ -1338,7 +1338,7 @@ dependencies = [ "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.24.2", "hyper-rustls 0.27.9", "hyper-util", @@ -1532,7 +1532,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "itoa", "matchit 0.8.4", @@ -1629,7 +1629,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.8.9", "object 0.37.3", "rustc-demangle", "windows-link 0.2.1", @@ -1824,6 +1824,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "blake2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "blake3" version = "1.8.7" @@ -1834,7 +1843,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq 0.4.2", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", ] [[package]] @@ -1856,6 +1865,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-modes" version = "0.8.1" @@ -1908,7 +1926,7 @@ dependencies = [ "hex", "http 1.5.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -2332,12 +2350,12 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -2390,7 +2408,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -2454,6 +2472,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cms" version = "0.2.3" @@ -2676,9 +2700,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -2818,6 +2842,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -2848,6 +2881,15 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curl-sys" version = "0.4.90+curl-8.21.0" @@ -3413,7 +3455,7 @@ dependencies = [ "arrow", "arrow-buffer", "base64 0.22.1", - "blake2", + "blake2 0.10.6", "blake3", "chrono", "datafusion-common", @@ -3953,7 +3995,7 @@ dependencies = [ "hickory-resolver", "http 1.5.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "ipnet", @@ -4156,7 +4198,7 @@ dependencies = [ "deno_error 0.6.1", "deno_tls", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "log", @@ -4455,10 +4497,21 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dirs" version = "4.0.0" @@ -5030,13 +5083,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "libz-sys", - "miniz_oxide", + "miniz_oxide 0.9.1", "zlib-rs", ] @@ -6109,7 +6162,7 @@ dependencies = [ "futures", "http 1.5.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.26.0", "hyper-tls", "hyper-tungstenite", @@ -6133,6 +6186,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -6159,9 +6221,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -6189,7 +6251,7 @@ dependencies = [ "futures-util", "headers", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "pin-project-lite", @@ -6205,7 +6267,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" dependencies = [ "hex", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6236,7 +6298,7 @@ checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "log", "rustls 0.22.4", @@ -6254,7 +6316,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "log", "rustls 0.23.35", @@ -6271,7 +6333,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6286,7 +6348,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "native-tls", "tokio", @@ -6301,7 +6363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a343d17fe7885302ed7252767dc7bb83609a874b6ff581142241ec4b73957ad" dependencies = [ "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6321,7 +6383,7 @@ dependencies = [ "futures-util", "http 1.5.0", "http-body 1.1.0", - "hyper 1.11.0", + "hyper 1.11.1", "ipnet", "libc", "percent-encoding", @@ -6342,7 +6404,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6946,7 +7008,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-http-proxy", "hyper-rustls 0.27.9", "hyper-timeout", @@ -7194,9 +7256,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" +checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96" dependencies = [ "bitflags 2.13.1", "libc", @@ -7328,9 +7390,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" +checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c" dependencies = [ "hashbrown 0.17.1", ] @@ -7701,6 +7763,15 @@ name = "miniz_oxide" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -7814,7 +7885,7 @@ dependencies = [ "futures-sink", "futures-util", "keyed_priority_queue", - "lru 0.18.2", + "lru 0.18.3", "mysql_common", "native-tls", "pem 3.0.6", @@ -8319,7 +8390,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "md-5 0.10.6", "parking_lot", @@ -8778,9 +8849,9 @@ dependencies = [ [[package]] name = "owo-colors" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" [[package]] name = "p12-keystore" @@ -8918,13 +8989,12 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.5.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", + "getrandom 0.4.3", + "phc", ] [[package]] @@ -9068,6 +9138,17 @@ dependencies = [ "phf 0.11.3", ] +[[package]] +name = "phc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +dependencies = [ + "base64ct", + "ctutils", + "getrandom 0.4.3", +] + [[package]] name = "phf" version = "0.11.3" @@ -10212,7 +10293,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-tls", "hyper-util", @@ -10260,7 +10341,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "js-sys", @@ -10316,7 +10397,7 @@ dependencies = [ "futures", "getrandom 0.2.17", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "reqwest 0.13.4", "reqwest-middleware", "retry-policies", @@ -13524,7 +13605,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -13556,7 +13637,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -13909,9 +13990,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.3" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" [[package]] name = "typed-path" @@ -14148,7 +14229,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -14280,7 +14361,7 @@ dependencies = [ "fslock", "gzip-header", "home", - "miniz_oxide", + "miniz_oxide 0.8.9", "paste", "which 6.0.3", ] @@ -14671,7 +14752,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-nats", @@ -14756,7 +14837,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.798.1" +version = "1.799.0" dependencies = [ "async-stream", "async-trait", @@ -14789,7 +14870,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14802,7 +14883,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "argon2", @@ -14832,7 +14913,7 @@ dependencies = [ "hex", "hmac", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "indexmap 2.14.0", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -14942,12 +15023,12 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "quick_cache", "serde", @@ -14965,7 +15046,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14982,7 +15063,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15008,7 +15089,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.798.1" +version = "1.799.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15018,7 +15099,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15035,7 +15116,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15057,7 +15138,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15080,7 +15161,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15096,11 +15177,11 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", - "hyper 1.11.0", + "hyper 1.11.1", "serde", "serde_json", "sql-builder", @@ -15118,7 +15199,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15139,7 +15220,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15153,7 +15234,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-nats", @@ -15188,14 +15269,14 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "axum 0.8.9", "base64 0.22.1", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "serde", "serde_json", @@ -15213,7 +15294,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15241,7 +15322,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15263,7 +15344,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15283,13 +15364,13 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", "futures", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "prometheus", @@ -15321,7 +15402,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15349,7 +15430,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.798.1" +version = "1.799.0" dependencies = [ "lazy_static", "serde", @@ -15361,13 +15442,13 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.798.1" +version = "1.799.0" dependencies = [ "argon2", "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "serde", "serde_json", @@ -15385,7 +15466,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15399,14 +15480,14 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.798.1" +version = "1.799.0" dependencies = [ "axum 0.8.9", "chrono", "futures", "hex", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "magic-crypt", "regex", @@ -15434,7 +15515,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.798.1" +version = "1.799.0" dependencies = [ "chrono", "lazy_static", @@ -15448,7 +15529,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15467,7 +15548,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.798.1" +version = "1.799.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15504,7 +15585,7 @@ dependencies = [ "globset", "hex", "hmac", - "hyper 1.11.0", + "hyper 1.11.1", "indexmap 2.14.0", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -15571,7 +15652,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.798.1" +version = "1.799.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -15590,7 +15671,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.798.1" +version = "1.799.0" dependencies = [ "regex", "serde", @@ -15605,7 +15686,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15629,7 +15710,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "futures", @@ -15646,7 +15727,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.798.1" +version = "1.799.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15662,7 +15743,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -15683,7 +15764,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -15714,7 +15795,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "arc-swap", @@ -15739,7 +15820,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-stream", @@ -15773,7 +15854,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "futures", @@ -15791,7 +15872,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.798.1" +version = "1.799.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15800,7 +15881,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -15812,7 +15893,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "serde_json", @@ -15824,7 +15905,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "gosyn", @@ -15836,7 +15917,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -15848,7 +15929,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "serde_json", @@ -15860,7 +15941,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "nu-parser", @@ -15871,7 +15952,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15882,7 +15963,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15894,7 +15975,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15905,7 +15986,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-recursion", @@ -15927,7 +16008,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "serde_json", @@ -15939,7 +16020,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -15953,7 +16034,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15970,7 +16051,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -15983,7 +16064,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "serde", @@ -15995,7 +16076,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -16013,7 +16094,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16029,7 +16110,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16045,7 +16126,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -16059,7 +16140,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-recursion", @@ -16098,7 +16179,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "const_format", @@ -16138,7 +16219,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.798.1" +version = "1.799.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16149,7 +16230,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-recursion", @@ -16160,7 +16241,7 @@ dependencies = [ "futures", "hex", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "magic-crypt", "quick_cache", @@ -16184,7 +16265,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -16208,14 +16289,14 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "rand 0.9.0", @@ -16241,7 +16322,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -16268,7 +16349,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -16301,7 +16382,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -16321,7 +16402,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -16355,7 +16436,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -16367,7 +16448,7 @@ dependencies = [ "hex", "hmac", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "matchit 0.7.3", @@ -16391,7 +16472,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -16414,7 +16495,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -16438,7 +16519,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-nats", @@ -16462,7 +16543,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -16497,7 +16578,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -16525,7 +16606,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-trait", @@ -16550,7 +16631,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16569,7 +16650,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-once-cell", @@ -16686,7 +16767,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.798.1" +version = "1.799.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 75cc6bc007..a05bd109ae 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.798.1" +version = "1.799.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.798.1" +version = "1.799.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index b6d83a9efb..47d807dd0d 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.798.1" +version = "1.799.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.798.1" +version = "1.799.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.798.1" +version = "1.799.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.798.1" +version = "1.799.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 7296524140..c4a544e545 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.798.1" +version = "1.799.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1a14a9fcf6..d2e036ac38 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.798.1 + version: 1.799.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 96c29cb209..b4f7ce465c 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.798.1"; +export const VERSION = "v1.799.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 3abd8b1e9a..c6aaf0b173 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.798.1"; +export const VERSION = "1.799.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0eed58d7d8..e6549e22ef 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.798.1", + "version": "1.799.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.798.1", + "version": "1.799.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -1755,6 +1755,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1771,6 +1772,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1787,6 +1789,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1803,6 +1806,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1819,6 +1823,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1835,6 +1840,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1851,6 +1857,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1867,6 +1874,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1883,6 +1891,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1899,6 +1908,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1915,6 +1925,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1931,6 +1942,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1947,6 +1959,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1963,6 +1976,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7569,7 +7583,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "devOptional": true, + "dev": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -8265,6 +8279,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8285,6 +8300,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8305,6 +8321,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8325,6 +8342,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8345,6 +8363,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8365,6 +8384,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8385,6 +8405,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8405,6 +8426,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8425,6 +8447,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8445,6 +8468,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8465,6 +8489,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -13170,6 +13195,21 @@ } } }, + "node_modules/svelte-check/node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -13949,7 +13989,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/frontend/package.json b/frontend/package.json index 895f218ba7..2fe0532761 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.798.1", + "version": "1.799.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index d22dabf5c7..22eb7440d2 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.798.1" +wmill = ">=1.799.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 338f03070f..38f43d1878 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.798.1 + version: 1.799.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index ea290b77d6..3cc5775a77 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.798.1' + ModuleVersion = '1.799.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index fb91df65df..2df87500db 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.798.1" +version = "1.799.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index e804a7ae57..1f0f5e2030 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.798.1", + "version": "1.799.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index d39061ec8d..a73353f780 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.798.1", + "version": "1.799.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 6cbed1aa9f..6e752179bd 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.798.1 +1.799.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index 64c7e322fb..678259b8ff 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.798.1", + "version": "1.799.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.798.1", + "version": "1.799.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index 5edfb07b4e..10a6f9918d 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.798.1", + "version": "1.799.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts", From 7c1a785f756ed27e4425f6534709b19971a73a97 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 29 Aug 2026 09:51:59 +0200 Subject: [PATCH 23/40] feat: serve service log retrieval from a columnar parquet store (#10886) * feat: always write service log files as json so they index structured Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016ijGGPCFkYhVzisFexAHYx * feat: serve service log retrieval from a columnar parquet store Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016ijGGPCFkYhVzisFexAHYx * feat: shrink the service log index to the per-host count it still serves Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016ijGGPCFkYhVzisFexAHYx * fix: reclaim the superseded service log index on upgrade Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016ijGGPCFkYhVzisFexAHYx * fix: address review findings in the service log store Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016ijGGPCFkYhVzisFexAHYx * chore: update ee-repo-ref to ad9e899dfd2ee4e3d18ecf06d016f821968c5a83 This commit updates the EE repository reference after PR #751 was merged in windmill-ee-private. Previous ee-repo-ref: 6ad4064f9d58d83612b42b4ec870384994d64bcb New ee-repo-ref: ad9e899dfd2ee4e3d18ecf06d016f821968c5a83 Automated by sync-ee-ref workflow. * fix: address review nits on the service log store Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016ijGGPCFkYhVzisFexAHYx --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/Cargo.lock | 3 + backend/ee-repo-ref.txt | 2 +- backend/src/monitor.rs | 3 +- backend/windmill-api/Cargo.toml | 8 +- backend/windmill-api/openapi.yaml | 30 ++++- backend/windmill-common/src/tracing_init.rs | 80 +++++------- backend/windmill-indexer/Cargo.toml | 11 +- backend/windmill-indexer/src/lib.rs | 2 + .../lib/components/ServiceLogsInner.svelte | 114 +++++++++++------- 9 files changed, 152 insertions(+), 101 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b7f278d437..028d42e9ce 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15693,9 +15693,11 @@ dependencies = [ "bytes", "chrono", "const_format", + "datafusion", "flume", "futures", "lazy_static", + "object_store", "serde", "serde_json", "sqlx", @@ -15703,6 +15705,7 @@ dependencies = [ "tempfile", "tokio", "tracing", + "url", "uuid", "windmill-common", "windmill-object-store", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index a1d9c874fc..5c23919dc8 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -9ff97cd818e85940fec282c92161e98c1b8583e2 +ad9e899dfd2ee4e3d18ecf06d016f821968c5a83 \ No newline at end of file diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 430f3659c3..6e0d50d30e 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -81,7 +81,6 @@ use windmill_common::{ jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, - tracing_init::JSON_FMT, users::truncate_token, utils::{empty_as_none, now_from_db, report_critical_error, Mode, HUB_API_SECRET}, worker::{ @@ -1373,7 +1372,7 @@ async fn send_log_file_to_object_store( match timeout(Duration::from_secs(10), sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8) ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", - hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT) + hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, true) .execute(db)).await { Ok(Ok(_)) => { if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 99a6cd90f7..ad7f12453c 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-amqp?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private", "windmill-api-npm-proxy/private"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-amqp?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private", "windmill-api-npm-proxy/private", "windmill-indexer?/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-debug/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-amqp?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-api-npm-proxy/enterprise", "license"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] @@ -18,10 +18,12 @@ agent_worker_server = ["dep:windmill-worker", "dep:windmill-api-agent-workers"] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] embedding = ["windmill-api-embeddings/embedding"] -parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "windmill-api-npm-proxy/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] +parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-indexer?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "windmill-api-npm-proxy/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus", "windmill-api-scripts/prometheus"] openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] -tantivy = ["dep:windmill-indexer"] +# The service log search handler reads the columnar store, so tantivy alone is +# not enough for this crate to build on its own. +tantivy = ["dep:windmill-indexer", "parquet"] kafka = ["dep:windmill-trigger-kafka", "windmill-store/kafka"] kafka-gssapi = ["kafka", "windmill-trigger-kafka/kafka-gssapi"] nats = ["dep:windmill-trigger-nats", "windmill-store/nats"] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index d2e036ac38..09a3cf4195 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -23890,7 +23890,7 @@ paths: items: type: string hits: - description: log files that matched the query + description: the log lines that matched the query, newest first type: array items: $ref: "#/components/schemas/LogSearchHit" @@ -34331,8 +34331,34 @@ components: LogSearchHit: type: object properties: - dancer: + ts: + description: timestamp of the log line itself, not of the file containing it type: string + format: date-time + host: + type: string + level: + type: string + enum: [TRACE, DEBUG, INFO, WARN, ERROR] + target: + description: the tracing target that emitted the line + type: string + nullable: true + message: + type: string + file_path: + description: the log file the line came from + type: string + line_no: + description: offset of the line within its file + type: integer + required: + - ts + - host + - level + - message + - file_path + - line_no AutoscalingEvent: type: object diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 5091c7e16d..b456e370fc 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -178,56 +178,40 @@ pub fn initialize_tracing( .with(logs_bridge.with_filter(otel_logs_filter)) .with(opentelemetry_filtered); - match *JSON_FMT { - true => { - // Stdout layer with its own filter - let stdout_layer = json_layer() - .with_writer(std::io::stdout) - .flatten_event(true) - .with_filter(stdout_env_filter) - .with_filter(create_targets_filter(default_env_filter)); + // The service log files are written to be indexed, not tailed, so they always carry the + // JSON format: it is what preserves level, target and the current span as fields rather + // than as text the index would have to recover by regex. JSON_FMT governs stdout only. + let file_layer = json_layer() + .with_writer(log_file_writer) + .flatten_event(true) + .with_filter(file_env_filter) + .with_filter(create_targets_filter(default_env_filter)); - // File layer with its own filter - let file_layer = json_layer() - .with_writer(log_file_writer) - .flatten_event(true) - .with_filter(file_env_filter) - .with_filter(create_targets_filter(default_env_filter)); + // Boxed so both arms have one type: the file layer is a single value and could not + // otherwise be typed against two different subscriber stacks. + let stdout_layer = match *JSON_FMT { + true => json_layer() + .with_writer(std::io::stdout) + .flatten_event(true) + .with_filter(stdout_env_filter) + .with_filter(create_targets_filter(default_env_filter)) + .boxed(), + false => compact_layer() + .with_writer(std::io::stdout) + .with_ansi(style.to_lowercase() != "never") + .with_file(true) + .with_line_number(true) + .with_target(false) + .with_filter(stdout_env_filter) + .with_filter(create_targets_filter(default_env_filter)) + .boxed(), + }; - base_layer - .with(stdout_layer) - .with(file_layer) - .with(CountingLayer::new()) - .init() - } - false => { - // Stdout layer with its own filter - let stdout_layer = compact_layer() - .with_writer(std::io::stdout) - .with_ansi(style.to_lowercase() != "never") - .with_file(true) - .with_line_number(true) - .with_target(false) - .with_filter(stdout_env_filter) - .with_filter(create_targets_filter(default_env_filter)); - - // File layer with its own filter - let file_layer = compact_layer() - .with_writer(log_file_writer) - .with_ansi(false) // No ANSI codes in log files - .with_file(true) - .with_line_number(true) - .with_target(false) - .with_filter(file_env_filter) - .with_filter(create_targets_filter(default_env_filter)); - - base_layer - .with(stdout_layer) - .with(file_layer) - .with(CountingLayer::new()) - .init() - } - } + base_layer + .with(stdout_layer) + .with(file_layer) + .with(CountingLayer::new()) + .init(); (_guard, meter_provider) } diff --git a/backend/windmill-indexer/Cargo.toml b/backend/windmill-indexer/Cargo.toml index acf90ab57a..09b2bbe5b1 100644 --- a/backend/windmill-indexer/Cargo.toml +++ b/backend/windmill-indexer/Cargo.toml @@ -10,7 +10,13 @@ path = "src/lib.rs" [features] default = [] -parquet = ["windmill-common/parquet", "windmill-object-store/parquet"] +parquet = [ + "windmill-common/parquet", + "windmill-object-store/parquet", + "dep:datafusion", + "dep:object_store", + "dep:url", +] private = ["windmill-common/private"] enterprise = ["windmill-common/enterprise", "windmill-object-store/enterprise"] @@ -33,3 +39,6 @@ astral-tokio-tar.workspace = true lazy_static.workspace = true const_format.workspace = true flume.workspace = true +datafusion = { workspace = true, optional = true } +object_store = { workspace = true, optional = true } +url = { workspace = true, optional = true } diff --git a/backend/windmill-indexer/src/lib.rs b/backend/windmill-indexer/src/lib.rs index 6c13b551d1..f107ea911d 100644 --- a/backend/windmill-indexer/src/lib.rs +++ b/backend/windmill-indexer/src/lib.rs @@ -7,3 +7,5 @@ pub mod indexer_oss; #[cfg(feature = "private")] pub mod service_logs_ee; pub mod service_logs_oss; +#[cfg(all(feature = "private", feature = "parquet"))] +pub mod service_logs_store_ee; diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte index bad18661b6..16bdd7d2ad 100644 --- a/frontend/src/lib/components/ServiceLogsInner.svelte +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -2,7 +2,7 @@ import { createBubbler, preventDefault } from 'svelte/legacy' const bubble = createBubbler() - import { IndexSearchService, ServiceLogsService } from '$lib/gen' + import { IndexSearchService, ServiceLogsService, type LogSearchHit } from '$lib/gen' import TimeframeSelect, { serviceLogsTimeframes, @@ -253,39 +253,50 @@ try { let res = '' log.split('\n').forEach((line) => { + // A file can hold both formats: the ones written before the layer + // switched to JSON, and panics or subprocess output that was never + // JSON to begin with. Those lines pass through as they are rather + // than being dropped, which would render the file blank. + let obj: any = undefined if (line.startsWith('{') && line.endsWith('}')) { - let obj = JSON.parse(line) - if (typeof obj == 'object') { - let nl = '' - if (obj['timestamp']) { - nl += obj['timestamp'] + ' ' - } - if (obj['level']) { - let lvl = obj['level'] - if (lvl == 'ERROR') { - nl += '\x1b[31mERROR\x1b[0m ' - } else if (lvl == 'INFO') { - nl += '\x1b[32mINFO\x1b[0m ' - } else { - nl += obj['level'] + ' ' - } - } - if (obj['message']) { - nl += obj['message'] + ' ' - } - delete obj['timestamp'] - delete obj['level'] - delete obj['message'] - Object.keys(obj).forEach((key) => { - nl += - key + - '=' + - (typeof obj[key] == 'object' ? JSON.stringify(obj[key]) : obj[key]) + - ' ' - }) - res += nl + '\n' + try { + obj = JSON.parse(line) + } catch { + obj = undefined } } + if (obj === null || typeof obj !== 'object') { + res += line + '\n' + } else { + let nl = '' + if (obj['timestamp']) { + nl += obj['timestamp'] + ' ' + } + if (obj['level']) { + let lvl = obj['level'] + if (lvl == 'ERROR') { + nl += '\x1b[31mERROR\x1b[0m ' + } else if (lvl == 'INFO') { + nl += '\x1b[32mINFO\x1b[0m ' + } else { + nl += obj['level'] + ' ' + } + } + if (obj['message']) { + nl += obj['message'] + ' ' + } + delete obj['timestamp'] + delete obj['level'] + delete obj['message'] + Object.keys(obj).forEach((key) => { + nl += + key + + '=' + + (typeof obj[key] == 'object' ? JSON.stringify(obj[key]) : obj[key]) + + ' ' + }) + res += nl + '\n' + } }) return res @@ -294,6 +305,23 @@ } } + // A hit is one log line with its fields already separated, so rendering it is + // formatting rather than parsing — there is no JSON to prettify and no + // snippet to highlight. + function renderHit(hit: LogSearchHit): string { + const level = + hit.level === 'ERROR' + ? '\x1b[31mERROR\x1b[0m' + : hit.level === 'WARN' + ? '\x1b[33mWARN\x1b[0m' + : hit.level === 'INFO' + ? '\x1b[32mINFO\x1b[0m' + : hit.level + return [hit.ts, level, hit.message, hit.target ? `target=${hit.target}` : ''] + .filter(Boolean) + .join(' ') + } + let logs: any = $state() let debounceTimeout: number | undefined = undefined @@ -399,7 +427,9 @@ ) { const res = await ServiceLogsService.getLogFile({ path: `${hostname}/${path}` }) - content = processLogWithJsonFmt(ansi_up.ansi_to_html(res), jsonFmt) + // Prettify first: it emits its own ANSI for the level, which converting + // beforehand would leave in the output as literal escapes. + content = ansi_up.ansi_to_html(processLogWithJsonFmt(res, jsonFmt)) hitLineNumber = lineNumber logDrawerOpen = true @@ -687,23 +717,19 @@ {:else if logs != undefined}
    - {#each logs.hits as { snippet_fragment, snippet_highlighted, document }} + + {#each logs.hits ?? [] as hit, i (`${i}:${hit.file_path}:${hit.line_no}`)} { - let logLineNumber = document.line_number[0] - let logFile = document.file_name[0] - let host = document.host[0] - let jsonFmt = document.json_fmt[0] - seeLogContext(logLineNumber, logFile, host, jsonFmt) - }} + content={renderHit(hit)} + highlighted={[]} + onClick={() => seeLogContext(hit.line_no, hit.file_path, hit.host, true)} /> {/each} - {#if logs.hits.length === 0} + {#if (logs.hits ?? []).length === 0}
    No logs
    {/if} - {#if logs.hits.length === 1000} + {#if (logs.hits ?? []).length === 1000}
    Older matches were truncated from this search, try refining your filters to get more precise results. From 419d3adb6c785a3fa8408f05a7f0b0f8d7f03920 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 29 Aug 2026 11:16:48 +0200 Subject: [PATCH 24/40] chore: bump tantivy to 0.27 and pin argon2 to 0.5 (#10890) * chore: bump tantivy to 0.27 and pin argon2 to 0.5 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016ijGGPCFkYhVzisFexAHYx * chore: pin tantivy to the merged fork main head --------- Co-authored-by: Claude Opus 5 (1M context) --- backend/Cargo.lock | 186 +++++++++++++-------------------------------- backend/Cargo.toml | 7 +- 2 files changed, 59 insertions(+), 134 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 028d42e9ce..7c8d40ad05 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -23,7 +23,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common 0.1.7", + "crypto-common", "generic-array", ] @@ -261,13 +261,13 @@ dependencies = [ [[package]] name = "argon2" -version = "0.6.0" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", - "blake2 0.11.0", - "cpufeatures 0.3.1", + "blake2", + "cpufeatures 0.2.17", "password-hash", ] @@ -1824,15 +1824,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "blake2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" -dependencies = [ - "digest 0.11.3", -] - [[package]] name = "blake3" version = "1.8.7" @@ -1865,15 +1856,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "block-buffer" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" -dependencies = [ - "hybrid-array", -] - [[package]] name = "block-modes" version = "0.8.1" @@ -2408,7 +2390,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.7", + "crypto-common", "inout", ] @@ -2472,12 +2454,6 @@ dependencies = [ "cc", ] -[[package]] -name = "cmov" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" - [[package]] name = "cms" version = "0.2.3" @@ -2842,15 +2818,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "crypto-common" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" -dependencies = [ - "hybrid-array", -] - [[package]] name = "csv" version = "1.4.0" @@ -2881,15 +2848,6 @@ dependencies = [ "cipher 0.4.4", ] -[[package]] -name = "ctutils" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" -dependencies = [ - "cmov", -] - [[package]] name = "curl-sys" version = "0.4.90+curl-8.21.0" @@ -3455,7 +3413,7 @@ dependencies = [ "arrow", "arrow-buffer", "base64 0.22.1", - "blake2 0.10.6", + "blake2", "blake3", "chrono", "datafusion-common", @@ -3731,9 +3689,9 @@ dependencies = [ [[package]] name = "datasketches" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" [[package]] name = "debug-helper" @@ -4497,21 +4455,10 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid", - "crypto-common 0.1.7", + "crypto-common", "subtle", ] -[[package]] -name = "digest" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" -dependencies = [ - "block-buffer 0.12.1", - "crypto-common 0.2.2", - "ctutils", -] - [[package]] name = "dirs" version = "4.0.0" @@ -5175,6 +5122,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "frostem" +version = "1.20260821.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ed6437a2ed7fc408115e25cdb41e15ba4b17742a4c8d3d1b5276fe9b687209" + [[package]] name = "fs3" version = "0.5.0" @@ -6186,15 +6139,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" -[[package]] -name = "hybrid-array" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" -dependencies = [ - "typenum", -] - [[package]] name = "hyper" version = "0.14.32" @@ -7379,15 +7323,6 @@ dependencies = [ "hashbrown 0.15.5", ] -[[package]] -name = "lru" -version = "0.16.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" -dependencies = [ - "hashbrown 0.16.1", -] - [[package]] name = "lru" version = "0.18.3" @@ -7423,9 +7358,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" [[package]] name = "lzma-sys" @@ -7849,9 +7784,9 @@ dependencies = [ [[package]] name = "murmurhash32" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" +checksum = "a8afc6df942f4c022d70c5725e18df1a705773870e5b7f96fde94ca0334ce77a" [[package]] name = "mysql-common-derive" @@ -8842,7 +8777,7 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "ownedbytes" version = "0.9.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "stable_deref_trait", ] @@ -8989,12 +8924,13 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.6.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ - "getrandom 0.4.3", - "phc", + "base64ct", + "rand_core 0.6.4", + "subtle", ] [[package]] @@ -9138,17 +9074,6 @@ dependencies = [ "phf 0.11.3", ] -[[package]] -name = "phc" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" -dependencies = [ - "base64ct", - "ctutils", - "getrandom 0.4.3", -] - [[package]] name = "phf" version = "0.11.3" @@ -10660,16 +10585,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rust_decimal" version = "1.42.1" @@ -12803,12 +12718,12 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" -version = "0.26.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.27.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "aho-corasick", "arc-swap", - "base64 0.22.1", + "base64 0.23.1", "bitpacking", "bon", "byteorder", @@ -12819,20 +12734,20 @@ dependencies = [ "downcast-rs", "fastdivide", "fnv", + "frostem", "fs4", "htmlescape", "itertools 0.14.0", "levenshtein_automata", "log", - "lru 0.16.4", - "lz4_flex 0.13.1", + "lru 0.18.3", + "lz4_flex 0.14.0", "measure_time", "memmap2", "once_cell", "oneshot", "rayon", "regex", - "rust-stemmers", "rustc-hash 2.1.3", "serde", "serde_json", @@ -12849,22 +12764,23 @@ dependencies = [ "thiserror 2.0.20", "time", "typetag", + "unwrap-infallible", "uuid", "winapi", ] [[package]] name = "tantivy-bitpacker" -version = "0.9.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.10.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "downcast-rs", "fastdivide", @@ -12878,8 +12794,8 @@ dependencies = [ [[package]] name = "tantivy-common" -version = "0.10.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.11.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "async-trait", "byteorder", @@ -12901,8 +12817,8 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" -version = "0.25.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.26.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "fnv", "nom", @@ -12913,8 +12829,8 @@ dependencies = [ [[package]] name = "tantivy-sstable" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "futures-util", "itertools 0.14.0", @@ -12926,8 +12842,8 @@ dependencies = [ [[package]] name = "tantivy-stacker" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "murmurhash32", "tantivy-common", @@ -12935,8 +12851,8 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "serde", ] @@ -14229,7 +14145,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common 0.1.7", + "crypto-common", "subtle", ] @@ -14257,6 +14173,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unwrap-infallible" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e497bb1f828cc9fb236722c2eaa100dcf201563f38f4da6252357a59037adf31" + [[package]] name = "ureq" version = "2.12.1" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a05bd109ae..3f614c9388 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -477,7 +477,10 @@ rust-embed = { version = "^6", features = ["interpolate-folder-path"] } mime_guess = "^2" hex = "^0" sql-builder = "^3" -argon2 = "^0" +# Pinned: `^0` floats to 0.6, which moved `password_hash::SaltString`, put +# `rand_core` behind a feature and changed `hash_password`'s signature — +# users_ee.rs is written against 0.5 and does not compile otherwise. +argon2 = "0.5" quick_cache = "^0" rand = "=0.9.0" rand_core = { version = "^0", features = ["std"] } @@ -697,7 +700,7 @@ tikv-jemalloc-ctl = { version = "^0.5" } triomphe = "^0" pin-project-lite = "^0" -tantivy = { git="https://github.com/windmill-labs/tantivy", rev="6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" } +tantivy = { git="https://github.com/windmill-labs/tantivy", rev="ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" } backon = "1.3.0" From c8172480b0b1be6c57210212afc71d6ec8711235 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 29 Aug 2026 11:53:21 +0200 Subject: [PATCH 25/40] fix: register every rotated service log file exactly once (#10891) * fix: register every rotated service log file exactly once Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016QTxWg4Sx57UodA9RpFMJm * chore: refresh sqlx cache for the log_file watermark query Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016QTxWg4Sx57UodA9RpFMJm * fix: skip service log files past the retention cutoff on catch-up Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016QTxWg4Sx57UodA9RpFMJm * refactor: name the shutdown flush for what it does and scope its doc claims Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016QTxWg4Sx57UodA9RpFMJm --------- Co-authored-by: Claude Opus 5 (1M context) --- ...01dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json | 22 + ...76dac75ba6df5f8eaa9f185300483e3ee36f.json} | 4 +- backend/src/main.rs | 4 +- backend/src/monitor.rs | 377 ++++++++++++------ 4 files changed, 281 insertions(+), 126 deletions(-) create mode 100644 backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json rename backend/.sqlx/{query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json => query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json} (51%) diff --git a/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json b/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json new file mode 100644 index 0000000000..4481fa3dea --- /dev/null +++ b/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT max(log_ts) FROM log_file\n WHERE hostname = $1 AND log_ts < (SELECT max(log_ts) FROM log_file WHERE hostname = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Timestamp" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c" +} diff --git a/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json b/backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json similarity index 51% rename from backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json rename to backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json index 7df22ca7b7..976cdfec06 100644 --- a/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json +++ b/backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", + "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", "describe": { "columns": [], "parameters": { @@ -17,5 +17,5 @@ }, "nullable": [] }, - "hash": "92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b" + "hash": "f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f" } diff --git a/backend/src/main.rs b/backend/src/main.rs index 69f28e11ee..8803157d84 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -14,7 +14,7 @@ use monitor::{ reload_nuget_config_setting, reload_powershell_repo_pat_setting, reload_powershell_repo_url_setting, reload_ruby_repos_setting, reload_timeout_wait_result_setting, reload_workspace_registries_setting, - send_current_log_file_to_object_store, send_logs_to_object_store, WORKERS_NAMES, + flush_pending_log_files_to_object_store, send_logs_to_object_store, WORKERS_NAMES, }; use rand::Rng; use sqlx::{Pool, Postgres}; @@ -1662,7 +1662,7 @@ Windmill Community Edition {GIT_VERSION} } else { tracing::info!("Nothing to do, exiting."); } - send_current_log_file_to_object_store(&conn, &hostname, &mode).await; + flush_pending_log_files_to_object_store(&conn, &hostname, &mode).await; if let Some(db) = conn.as_sql() { tracing::info!("Exiting connection pool"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 6e0d50d30e..d640e1bda6 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1220,32 +1220,60 @@ async fn sleep_until_next_minute_start_plus_one_s() { } use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE; -async fn find_two_highest_files(hostname: &str) -> (Option, Option) { + +/// The minutely rolling appender names each file `.log.<%Y-%m-%d-%H-%M>`; +/// anything else in the directory is not a rotated log file. +fn parse_log_file_ts(file_name: &str) -> Option { + NaiveDateTime::parse_from_str( + file_name.rsplit('.').next()?, + windmill_common::tracing_init::LOG_TIMESTAMP_FMT, + ) + .ok() +} + +/// Oldest first. Readdir order is filesystem-dependent — tmpfs hands back the +/// newest entry first, ext4 hashes the names — so the listing has to be sorted +/// before anything picks a file out of it. +fn sorted_log_files(file_names: impl Iterator) -> Vec<(NaiveDateTime, String)> { + let mut files = file_names + .filter_map(|name| parse_log_file_ts(&name).map(|ts| (ts, name))) + .collect::>(); + files.sort(); + files +} + +/// Every log file but the newest one: that one is still being appended to, every +/// older one is final. +fn rotated_log_files(file_names: impl Iterator) -> Vec<(NaiveDateTime, String)> { + let mut files = sorted_log_files(file_names); + files.pop(); + files +} + +async fn read_log_file_names(hostname: &str) -> Vec { let log_dir = format!("{}/{}/", *TMP_WINDMILL_LOGS_SERVICE, hostname); - let rd_dir = tokio::fs::read_dir(log_dir).await; - if let Ok(mut log_files) = rd_dir { - let mut highest_file: Option = None; - let mut second_highest_file: Option = None; - while let Ok(Some(file)) = log_files.next_entry().await { - let file_name = file - .file_name() - .to_str() - .map(|x| x.to_string()) - .unwrap_or_default(); - if file_name > highest_file.clone().unwrap_or_default() { - second_highest_file = highest_file; - highest_file = Some(file_name); - } + let mut rd_dir = match tokio::fs::read_dir(&log_dir).await { + Ok(rd_dir) => rd_dir, + Err(e) => { + tracing::error!("Error reading log files: {}, {:#?}", log_dir, e); + return vec![]; + } + }; + let mut file_names = vec![]; + while let Ok(Some(file)) = rd_dir.next_entry().await { + if let Some(file_name) = file.file_name().to_str() { + file_names.push(file_name.to_string()); } - (highest_file, second_highest_file) - } else { - tracing::error!( - "Error reading log files: {}, {:#?}", - *TMP_WINDMILL_LOGS_SERVICE, - rd_dir.unwrap_err() - ); - (None, None) } + file_names +} + +async fn list_log_files(hostname: &str) -> Vec<(NaiveDateTime, String)> { + sorted_log_files(read_log_file_names(hostname).await.into_iter()) +} + +async fn list_rotated_log_files(hostname: &str) -> Vec<(NaiveDateTime, String)> { + rotated_log_files(read_log_file_names(hostname).await.into_iter()) } fn get_worker_group(mode: &Mode) -> Option { @@ -1265,133 +1293,187 @@ pub fn send_logs_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(10)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + init_last_log_file_sent(&conn, &hostname).await; sleep_until_next_minute_start_plus_one_s().await; loop { interval.tick().await; - let (_, snd_highest_file) = find_two_highest_files(&hostname).await; - send_log_file_to_object_store( - &hostname, - &mode, - &worker_group, - &conn, - snd_highest_file, - false, - ) - .await; + let files = list_rotated_log_files(&hostname).await; + send_log_files_to_object_store(&hostname, &mode, &worker_group, &conn, files).await; } }); } -pub async fn send_current_log_file_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) { - tracing::info!("Sending current log file to object store"); - let (highest_file, _) = find_two_highest_files(hostname).await; +pub async fn flush_pending_log_files_to_object_store( + conn: &Connection, + hostname: &str, + mode: &Mode, +) { + tracing::info!("Sending pending log files to object store"); let worker_group = get_worker_group(&mode); - send_log_file_to_object_store(hostname, mode, &worker_group, conn, highest_file, true).await; -} - -fn get_now_and_str() -> (NaiveDateTime, String) { - let ts = Utc::now().naive_utc(); - ( - ts, - ts.format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT) - .to_string(), - ) + // Nothing rotates after this, so the file still being appended to is registered + // here, along with any rotated one the loop had not reached yet. Bounded like the + // pool close that follows: a backlog against a slow object store would otherwise + // hold the process past its termination grace period. Whatever is left over is + // registered by the next run's catch-up. + let flush = async { + let files = list_log_files(hostname).await; + send_log_files_to_object_store(hostname, mode, &worker_group, conn, files).await; + }; + if timeout(Duration::from_secs(15), flush).await.is_err() { + tracing::warn!("Could not send all pending log files in time (15s). Exiting anyway."); + } } lazy_static::lazy_static! { static ref LAST_LOG_FILE_SENT: Arc>> = Arc::new(Mutex::new(None)); + /// Serializes the periodic uploader against the shutdown flush. The uploader is a + /// detached task that keeps ticking while the flush runs and both walk the same + /// files, so without this both can clear the watermark for one file and count its + /// lines twice through the additive upsert. + static ref SENDING_LOG_FILES: tokio::sync::Mutex<()> = tokio::sync::Mutex::new(()); } +fn last_log_file_sent() -> Option { + LAST_LOG_FILE_SENT.lock().ok().and_then(|ts| *ts) +} + +/// Resume from what this host already registered, so a previous run's leftovers reach +/// the object store rather than being dropped. Their line counts come out zero, this +/// run having counted none of them, which only flattens their bars in the UI. +/// +/// The newest registered minute is left out on purpose: the shutdown flush registers +/// the file that was still open and the appender reopens that minute in append mode, +/// so a restart inside it would otherwise strand everything written afterwards. +/// +/// A row rewritten this way restores the object and sums the counters, but whether the +/// indexers read it again depends on their single `log_ts >` cursor, which is not +/// per-hostname: a minute at or below it stays out of search until it is re-indexed. +async fn init_last_log_file_sent(conn: &Connection, hostname: &str) { + let Some(db) = conn.as_sql() else { + return; + }; + match sqlx::query_scalar!( + "SELECT max(log_ts) FROM log_file + WHERE hostname = $1 AND log_ts < (SELECT max(log_ts) FROM log_file WHERE hostname = $1)", + hostname + ) + .fetch_one(db) + .await + { + Ok(Some(ts)) => { + if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { + last_log_file_sent.replace(ts); + }) { + tracing::error!("Error initializing last log file sent: {:?}", e); + } + } + Ok(None) => {} + Err(e) => tracing::error!("Error loading last log file sent: {:?}", e), + } +} + +async fn send_log_files_to_object_store( + hostname: &str, + mode: &Mode, + worker_group: &Option, + conn: &Connection, + files: Vec<(NaiveDateTime, String)>, +) { + let _guard = SENDING_LOG_FILES.lock().await; + let retention_cutoff = + Utc::now().naive_utc() - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS); + for (ts, file_name) in files { + if last_log_file_sent().is_some_and(|last| last >= ts) { + continue; + } + // A run coming back from a long outage still finds its predecessor's files on + // disk. Registering one past the retention cutoff inserts a row + // `delete_expired_items` drops on its next pass, once the indexers have already + // paid to parse it. + if ts < retention_cutoff { + continue; + } + // Stop at the first failure rather than moving on: both indexers walk + // `log_file` with a `log_ts > watermark` cursor, so a row that lands after + // a newer one is never picked up. + if !send_log_file_to_object_store(hostname, mode, worker_group, conn, &file_name, ts).await + { + break; + } + } +} + +/// Returns whether the file ended up registered in `log_file`. async fn send_log_file_to_object_store( hostname: &str, mode: &Mode, worker_group: &Option, conn: &Connection, - snd_highest_file: Option, - use_now: bool, -) { - if let Some(highest_file) = snd_highest_file { - //parse datetime frome file xxxx.yyyy-MM-dd-HH-mm - let (ts, ts_str) = if use_now { - get_now_and_str() - } else { - highest_file - .split(".") - .last() - .and_then(|x| { - NaiveDateTime::parse_from_str( - x, - windmill_common::tracing_init::LOG_TIMESTAMP_FMT, - ) - .ok() - .map(|y| (y, x.to_string())) - }) - .unwrap_or_else(get_now_and_str) - }; + file_name: &str, + ts: NaiveDateTime, +) -> bool { + #[cfg(feature = "parquet")] + if let Some(s3_client) = windmill_object_store::get_object_store().await { + let path = std::path::Path::new(&*TMP_WINDMILL_LOGS_SERVICE) + .join(hostname) + .join(file_name); - let exists = LAST_LOG_FILE_SENT.lock().map(|last_log_file_sent| { - last_log_file_sent - .map(|last_log_file_sent| last_log_file_sent >= ts) - .unwrap_or(false) - }); - - if exists.unwrap_or(false) { - return; - } - - #[cfg(feature = "parquet")] - let s3_client = windmill_object_store::get_object_store().await; - #[cfg(feature = "parquet")] - if let Some(s3_client) = s3_client { - let path = std::path::Path::new(&*TMP_WINDMILL_LOGS_SERVICE) - .join(hostname) - .join(&highest_file); - - //read file as byte stream - let bytes = tokio::fs::read(&path).await; - if let Err(e) = bytes { + //read file as byte stream + let bytes = match tokio::fs::read(&path).await { + Ok(bytes) => bytes, + Err(e) => { tracing::error!("Error reading log file: {:?}", e); - return; + return false; } - let path = windmill_object_store::object_store_reexports::Path::from_url_path(format!( - "{}{hostname}/{highest_file}", - windmill_common::tracing_init::LOGS_SERVICE - )); - if let Err(e) = path { + }; + let path = windmill_object_store::object_store_reexports::Path::from_url_path(format!( + "{}{hostname}/{file_name}", + windmill_common::tracing_init::LOGS_SERVICE + )); + let path = match path { + Ok(path) => path, + Err(e) => { tracing::error!("Error creating log file path: {:?}", e); - return; - } - if let Err(e) = s3_client.put(&path.unwrap(), bytes.unwrap().into()).await { - tracing::error!("Error sending logs to object store: {:?}", e); + return false; } + }; + if let Err(e) = s3_client.put(&path, bytes.into()).await { + tracing::error!("Error sending logs to object store: {:?}", e); + return false; } + } - let (ok_lines, err_lines) = read_log_counters(ts_str); + let ts_str = ts + .format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT) + .to_string(); + let (ok_lines, err_lines) = read_log_counters(ts_str); - if let Some(db) = conn.as_sql() { - match timeout(Duration::from_secs(10), sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) - VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8) - ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", - hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, true) - .execute(db)).await { - Ok(Ok(_)) => { - if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { - last_log_file_sent.replace(ts); - }) { - tracing::error!("Error updating last log file sent: {:?}", e); - } - tracing::info!("Log file sent: {}", highest_file); - } - Ok(Err(e)) => { - tracing::error!("Error inserting log file: {:?}", e); - } - Err(e) => { - tracing::error!("Error inserting log file, timeout elapsed: {:?}", e); - } + let Some(db) = conn.as_sql() else { + // not sending log file to object store in agent mode + return false; + }; + + match timeout(Duration::from_secs(10), sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) + VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8) + ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", + hostname, mode.to_string(), worker_group.clone(), ts, file_name, ok_lines as i64, err_lines as i64, true) + .execute(db)).await { + Ok(Ok(_)) => { + if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { + last_log_file_sent.replace(ts); + }) { + tracing::error!("Error updating last log file sent: {:?}", e); } - } else { - // tracing::warn!("Not sending log file to object store in agent mode"); - () + tracing::info!("Log file sent: {}", file_name); + true + } + Ok(Err(e)) => { + tracing::error!("Error inserting log file: {:?}", e); + false + } + Err(e) => { + tracing::error!("Error inserting log file, timeout elapsed: {:?}", e); + false } } } @@ -6831,3 +6913,54 @@ mod zombie_worker_memory_pct_tests { ); } } + +#[cfg(test)] +mod log_file_listing_tests { + use super::{rotated_log_files, sorted_log_files}; + + fn names(files: Vec<(chrono::NaiveDateTime, String)>) -> Vec { + files.into_iter().map(|(_, n)| n).collect() + } + + /// A directory read newest-entry-first is what tmpfs actually hands back. + #[test] + fn orders_by_minute_whatever_order_readdir_used() { + let newest_first = [ + "h.log.2026-08-29-06-49", + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-48", + "h.log.2026-08-29-06-47", + ]; + assert_eq!( + names(sorted_log_files(newest_first.iter().map(|x| x.to_string()))), + vec![ + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-47", + "h.log.2026-08-29-06-48", + "h.log.2026-08-29-06-49", + ] + ); + assert_eq!( + names(rotated_log_files( + newest_first.iter().map(|x| x.to_string()) + )), + vec![ + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-47", + "h.log.2026-08-29-06-48", + ] + ); + } + + #[test] + fn drops_names_that_are_not_rotated_log_files() { + let files = sorted_log_files( + ["h.log", "not-a-log-file", "h.log.2026-08-29-06-46"] + .iter() + .map(|x| x.to_string()), + ); + assert_eq!(files.len(), 1); + assert_eq!(files[0].1, "h.log.2026-08-29-06-46"); + assert_eq!(files[0].0.to_string(), "2026-08-29 06:46:00"); + } +} From 338d75cc5227e352cb84828c99bfd3b984cf0fa5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 29 Aug 2026 18:47:53 +0200 Subject: [PATCH 26/40] feat: serve service log context from parquet and retire the raw log files (#10892) * feat: serve service log context from the parquet store and retire the raw files * fix: keep the log ingest cursor in the store and stream file rebuilds * fix: roll back a partial index rebuild and move the cursor before the commit * fix: make the index rebuild idempotent and repair a cursor the index never caught up with * fix: seed the indexed cursor on upgrade and after a rebuild * fix: fail the indexing pass on an unreadable cursor instead of reading it as absent * docs: record what keeps both known_ts entries, not the path main removed * chore: update ee-repo-ref to 466eb1830879052a5d042295256a78375bee916d This commit updates the EE repository reference after PR #754 was merged in windmill-ee-private. Previous ee-repo-ref: ddb3a536b8d85c134c01f87da7783baaa204a6d1 New ee-repo-ref: 466eb1830879052a5d042295256a78375bee916d Automated by sync-ee-ref workflow. --------- Co-authored-by: windmill-internal-app[bot] --- ...34f1befe6b16eee363c843faa1f836d53ca8d.json | 29 +++++ backend/ee-repo-ref.txt | 2 +- backend/windmill-api/src/service_logs.rs | 104 +++++++++++++++--- 3 files changed, 116 insertions(+), 19 deletions(-) create mode 100644 backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json diff --git a/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json b/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json new file mode 100644 index 0000000000..d64f007d7b --- /dev/null +++ b/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT mode::text AS \"mode!\", log_ts FROM log_file WHERE hostname = $1 AND file_path = $2 ORDER BY log_ts DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "mode!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "log_ts", + "type_info": "Timestamp" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 5c23919dc8..cce83bbdff 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ad9e899dfd2ee4e3d18ecf06d016f821968c5a83 \ No newline at end of file +466eb1830879052a5d042295256a78375bee916d diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs index 99dc7c41a8..a54f275f46 100644 --- a/backend/windmill-api/src/service_logs.rs +++ b/backend/windmill-api/src/service_logs.rs @@ -88,6 +88,66 @@ async fn list_files( Ok(Json(rows)) } +/// Rebuild one source log file from the columnar store. +/// +/// Not the original bytes: the store holds a line's fields rather than its text, +/// so the JSON is re-serialized here and key order and whitespace are this +/// writer's. Everything a reader can see survives — the drawer this feeds +/// renders a prettified view of each line either way, and a line that was never +/// JSON comes back exactly as it was written. +#[cfg(all(feature = "tantivy", feature = "private"))] +async fn get_log_file_from_store( + db: &DB, + store: &windmill_indexer::service_logs_store_ee::Store, + path: &str, +) -> windmill_common::error::Result { + let (hostname, file_name) = path + .split_once('/') + .ok_or_else(|| Error::BadRequest("Invalid path".to_string()))?; + + // The store is partitioned by day and mode, neither of which the path + // carries. `log_file` names both, and its primary key starts with hostname. + let file = sqlx::query!( + // `mode!` because the column is NOT NULL and only the cast makes sqlx + // think otherwise; a silent default would look up a `mode=` partition + // that matches nothing and read as a missing file. + "SELECT mode::text AS \"mode!\", log_ts FROM log_file WHERE hostname = $1 AND file_path = $2 ORDER BY log_ts DESC LIMIT 1", + hostname, + file_name + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("File {path} not found")))?; + + // A row registered by this version carries the minute in the file's own name, + // so the two agree and the second is redundant. One written before the + // uploader derived `log_ts` from the name carries a wall clock instead, and + // those outlive an upgrade by the retention period — which is also what makes + // the `ORDER BY` above worth having. The name is authoritative, so both go. + let mut known_ts = vec![chrono::DateTime::from_naive_utc_and_offset( + file.log_ts, + chrono::Utc, + )]; + if let Some(named) = file_name.rsplit('.').next().and_then(|s| { + chrono::NaiveDateTime::parse_from_str(s, windmill_common::tracing_init::LOG_TIMESTAMP_FMT) + .ok() + }) { + known_ts.push(chrono::DateTime::from_naive_utc_and_offset( + named, + chrono::Utc, + )); + } + + let text = windmill_indexer::service_logs_store_ee::read_log_file( + store, &file.mode, hostname, file_name, &known_ts, + ) + .await + .map_err(|e| Error::internal_err(format!("Error reading the service log store: {e}")))? + .ok_or_else(|| Error::NotFound(format!("File {path} not found")))?; + + Ok(content_plain(Body::from(text))) +} + async fn get_log_file( authed: ApiAuthed, Extension(db): Extension, @@ -104,27 +164,30 @@ async fn get_log_file( let s3_client = windmill_object_store::get_object_store().await; #[cfg(feature = "parquet")] if let Some(s3_client) = s3_client { - let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); - let file = s3_client + use windmill_object_store::object_store_reexports::ObjectStoreError; + + // The raw file, for as long as it is there. It outlives its ingestion by + // one indexer pass at most, so this covers the most recent minutes of a + // host's logs byte for byte; everything older is rebuilt from the store. + let object_path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); + match s3_client .get(&windmill_object_store::object_store_reexports::Path::from( - path, + object_path, )) - .await; - match file { - Ok(file) => { - let bytes = file.bytes().await; - match bytes { - Ok(bytes) => { - return Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))); - } - Err(e) => { - return Err(Error::internal_err(format!( - "Error pulling the bytes: {}", - e - ))); - } + .await + { + Ok(file) => match file.bytes().await { + Ok(bytes) => { + return Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))); } - } + Err(e) => { + return Err(Error::internal_err(format!( + "Error pulling the bytes: {}", + e + ))); + } + }, + Err(ObjectStoreError::NotFound { .. }) => {} Err(e) => { return Err(Error::internal_err(format!( "Error fetching the file: {}", @@ -132,6 +195,11 @@ async fn get_log_file( ))); } } + + #[cfg(all(feature = "tantivy", feature = "private"))] + return get_log_file_from_store(&db, &s3_client, &path).await; + #[cfg(not(all(feature = "tantivy", feature = "private")))] + return Err(Error::NotFound(format!("File {path} not found"))); } let full_path = format!("{}{}", *TMP_WINDMILL_LOGS_SERVICE, path); // SECURITY (defense in depth): refuse to read through a symlink so a planted From 815de49e2322f85ca92b1e41a2bcd22591ebe93f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 29 Aug 2026 19:39:14 +0200 Subject: [PATCH 27/40] feat: make the service log retention period an instance setting (#10889) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: make the service log retention period an instance setting Service log retention was a hardcoded 14 days with no override, unlike job retention. It becomes the `service_log_retention_secs` global setting (env `SERVICE_LOG_RETENTION_SECS`, default unchanged at 14 days), reloaded on change like the other retention settings. The constant becomes `DEFAULT_SERVICE_LOG_RETENTION_SECS` and every reader goes through `service_log_retention_secs()`, so the `log_file` sweep, the object-storage orphan scan, the columnar store's compaction and pruning, the retrieval clamp and the search index's trim window all follow the configured value. Loaded outside `initial_load`'s `server_mode` guard: a dedicated indexer trims the search index to a window derived from this value and is not a server. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN * fix: never let a non-positive service log retention expire every log Every service log cutoff is `now - retention`, so a `0` or negative window puts the cutoff at or after `now` and the next sweep reads the whole history as expired — deleting the `log_file` rows and their object-storage files irreversibly. `0` is reachable two ways now that the window is configurable: it is what an operator types by analogy with the job retention period sitting directly above it, where `0` does mean keep forever; and `SecondsInput` writes a `0` into a field that was merely focused, so saving the Jobs panel is enough. Service logs always have a window, so clamp an unusable value back to the default in the accessor every reader already goes through. The upper bound is where `chrono::Duration::seconds` panics, which would abort the sweep that reads it. The settings field rejects a non-positive value rather than silently correcting it, and its description now names the database rows too — they are swept on every instance, including one with no object storage configured. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN * fix: address review findings on the service log retention setting - Bound the monitor's `log_file` sweep. Every process rotates a log file a minute, so lowering the retention can make one ordinary setting change expire millions of rows; the unbounded `DELETE ... RETURNING` materialized all of them, and their deletion futures, in a single tick. Batched like the settings-page cleanup on the same table. - Make the retention atomic private and give it one writer, so a value that would expire every service log cannot reach a cutoff by any path, and say so in the log when one is rejected rather than falling back silently. - Cap the retention at a century. The previous ceiling only bounded `TimeDelta` construction, while consumers compute `now - retention`, which panics past year 262143, and build a Postgres interval that overflows well before the old cap. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN * fix: cap an oversized service log retention instead of shortening it The two unusable directions were landing on the same fallback, so configuring a retention above the ceiling silently produced 14 days — deleting logs the operator had asked to keep for longer. Too large now caps at the maximum, which preserves that intent; only a non-positive value, which would expire everything and has no upward reading, falls back to the default. Also bound the `log_file` drain to ten batches per pass: `monitor_db` runs under a 600s timeout that cancels every maintenance future in the same `join!` and reports a critical error, so a backlog large enough to need batching has to drain across ticks, the way the neighbouring sweeps already do. The settings field carries the upper bound too, and the superseded query's offline entry is dropped. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN * fix: route the new log-file registration cutoff through the retention accessor `send_log_files_to_object_store` arrived on main while this branch was open and reads the retention directly. The atomic behind it is private now, so it goes through the accessor like every other consumer — which also means the cutoff it uses to skip registering already-expired files follows the configured retention rather than a fixed two weeks. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN * fix: say why every mode loads the service log retention setting A worker registers its rotated log files against the retention cutoff, so the comment naming only the indexer no longer covers why the setting sits outside the `server_mode` guard. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN * fix: file service log retention under Monitoring, not Jobs Service logs are the Windmill processes' own logs — every process rotates and registers its own, no job involved — so the Jobs panel was grouping by the shape of the widget rather than by the subject. It sits under Monitoring now, beside the Indexer panel that holds the other service-log window. Its own section rather than inside that panel: the panel is badged EE, while this governs the database sweep that runs on every instance. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN * chore: update ee-repo-ref to a6e3533b26195918a17fea58646f71d2bbcde288 This commit updates the EE repository reference after PR #752 was merged in windmill-ee-private. Previous ee-repo-ref: 1d93da24bd166b9a5a5cc204034a1d35ffc88474 New ee-repo-ref: a6e3533b26195918a17fea58646f71d2bbcde288 Automated by sync-ee-ref workflow. * feat: say on the service logs page where the logs actually are The retention number alone does not tell an operator what it governs, and the answer differs by instance. Two states are worth calling out because they are the ones where retention does not mean what it looks like: Without instance object storage, each process keeps its files on its own disk. The page lists what every host wrote, since the rows are in the shared database, but can only open the files of the replica serving the request, and a host's files go with it when it is replaced. With object storage but "Delete logs from s3 periodically" off — the backend default, since uploads are gated on a store existing while deletions are gated on that toggle — expiring a log removes the row and the local file and leaves the uploaded copy behind for good. The retention field itself now names every copy it covers and says that full-text search reaches back at most that far, and less when the indexer's own window is shorter. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN * fix: describe raw log files as the transient copy they became Retiring the raw files landed while this was being written: the indexer now deletes each one as soon as it is ingested, and the log viewer rebuilds a file from the columnar store once the raw copy is gone. So the durable copy is the store, and warning that an uploaded file is kept forever when periodic s3 deletion is off only holds where no indexer runs to ingest it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN * chore: point ee-repo-ref at the EE compile fix EE main does not build on its own: extracting the index-window expression and adding a fourth copy of it landed in separate PRs that never conflicted textually. windmill-ee-private#756 is the one-line fix; this pins it so CI has a tree that compiles. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- ...90c4b8506acefc85ef5b2f92a7bc451b1c5e.json} | 5 +- backend/ee-repo-ref.txt | 2 +- backend/src/main.rs | 11 +- backend/src/monitor.rs | 106 ++++++++++++++---- .../windmill-api-settings/src/log_cleanup.rs | 13 ++- .../windmill-common/src/global_settings.rs | 1 + backend/windmill-common/src/indexer.rs | 72 ++++++++++++ backend/windmill-common/src/lib.rs | 50 ++++++++- .../lib/components/InstanceSettings.svelte | 26 +++++ .../src/lib/components/instanceSettings.ts | 25 +++++ 10 files changed, 275 insertions(+), 36 deletions(-) rename backend/.sqlx/{query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json => query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json} (51%) diff --git a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json similarity index 51% rename from backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json rename to backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json index 24a5ee62dd..42bac71afd 100644 --- a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json +++ b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname", + "query": "DELETE FROM log_file WHERE (hostname, log_ts) IN (\n SELECT hostname, log_ts FROM log_file\n WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval\n LIMIT $2\n ) RETURNING file_path, hostname", "describe": { "columns": [ { @@ -16,6 +16,7 @@ ], "parameters": { "Left": [ + "Int8", "Int8" ] }, @@ -24,5 +25,5 @@ false ] }, - "hash": "94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033" + "hash": "0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index cce83bbdff..6f56f0ab9a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -466eb1830879052a5d042295256a78375bee916d +e6483ff5a289521405912d95be5c5ba064bedb38 diff --git a/backend/src/main.rs b/backend/src/main.rs index 8803157d84..c782e1ad97 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -61,8 +61,9 @@ use windmill_common::{ SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, + UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING, @@ -143,7 +144,8 @@ use crate::monitor::{ reload_pip_index_url_setting, reload_retention_period_setting, reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting, - reload_sandbox_registry_auth_setting, reload_scim_token_setting, reload_smtp_config, + reload_sandbox_registry_auth_setting, reload_scim_token_setting, + reload_service_log_retention_secs_setting, reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting, reload_worker_config, MonitorIteration, @@ -1953,6 +1955,9 @@ async fn process_notify_event( } TIMEOUT_WAIT_RESULT_SETTING => reload_timeout_wait_result_setting(conn).await, RETENTION_PERIOD_SECS_SETTING => reload_retention_period_setting(conn).await, + SERVICE_LOG_RETENTION_SECS_SETTING => { + reload_service_log_retention_secs_setting(conn).await + } RETENTION_PERIOD_SECS_OVERRIDES_SETTING => { if let Err(e) = load_retention_period_overrides(db).await { tracing::error!("Error loading per-workspace retention overrides: {e:#}"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index d640e1bda6..c0748a3798 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -70,11 +70,11 @@ use windmill_common::{ RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, - WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, - WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, - WORKSPACE_MAX_QUEUED_JOBS_SETTING, + SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, + UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, + WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, + WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, }, indexer::load_indexer_config, jobs::delete_jobs, @@ -97,10 +97,10 @@ use windmill_common::{ KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, - HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, - JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, METRICS_ENABLED, - MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, - SERVICE_LOG_RETENTION_SECS, STORE_AUDIT_LOGS_S3, + DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, JOB_RETENTION_SECS, + JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, + METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, + OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3, }; use windmill_common::{ client::AuthedClient, @@ -475,6 +475,19 @@ pub async fn initial_load( |v: Option| async move { HUB_API_SECRET.store(std::sync::Arc::new(v)) }, ); + // Outside the `server_mode` guard below: every mode reads this. A worker registers its + // rotated log files against the cutoff, and a dedicated indexer trims the search index to a + // window derived from it — neither is a server. + pass.setting(SERVICE_LOG_RETENTION_SECS_SETTING, true, |v| async move { + windmill_common::set_service_log_retention_secs(parse_setting_value::( + v, + SERVICE_LOG_RETENTION_SECS_SETTING, + "SERVICE_LOG_RETENTION_SECS", + DEFAULT_SERVICE_LOG_RETENTION_SECS, + |x| x, + )) + }); + if server_mode { pass.setting(RETENTION_PERIOD_SECS_SETTING, true, |v| async move { JOB_RETENTION_SECS.store( @@ -1380,8 +1393,8 @@ async fn send_log_files_to_object_store( files: Vec<(NaiveDateTime, String)>, ) { let _guard = SENDING_LOG_FILES.lock().await; - let retention_cutoff = - Utc::now().naive_utc() - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS); + let retention_cutoff = Utc::now().naive_utc() + - chrono::Duration::seconds(windmill_common::service_log_retention_secs()); for (ts, file_name) in files { if last_log_file_sent().is_some_and(|last| last >= ts) { continue; @@ -1661,6 +1674,13 @@ pub async fn trim_resource_versions(db: &DB) -> () { } } +/// Matches the batch the settings-page cleanup uses for the same table. +const SERVICE_LOG_DELETE_BATCH: i64 = 2_000; +/// Batches per pass. `monitor_db` runs under a 600s timeout that cancels every maintenance +/// future in the same `join!` and reports a critical error, so a large backlog has to drain +/// across ticks rather than inside one, the way the neighbouring sweeps already do. +const SERVICE_LOG_DELETE_MAX_BATCHES: usize = 10; + pub async fn delete_expired_items(db: &DB) -> () { let expired_tokens_r = sqlx::query_as!( TokenRow, @@ -1743,23 +1763,48 @@ pub async fn delete_expired_items(db: &DB) -> () { Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()), } - match sqlx::query_as!( - LogFile, - "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname", - SERVICE_LOG_RETENTION_SECS, - ) - .fetch_all(db) - .await - { - Ok(log_files_to_delete) => { + // Batched: every process rotates a log file a minute, so lowering the retention makes one + // ordinary setting change expire millions of rows at once. An unbounded `DELETE ... + // RETURNING` would materialize all of them, and their deletion futures, in this one tick. + for _ in 0..SERVICE_LOG_DELETE_MAX_BATCHES { + let batch = sqlx::query_as!( + LogFile, + "DELETE FROM log_file WHERE (hostname, log_ts) IN ( + SELECT hostname, log_ts FROM log_file + WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval + LIMIT $2 + ) RETURNING file_path, hostname", + windmill_common::service_log_retention_secs(), + SERVICE_LOG_DELETE_BATCH, + ) + .fetch_all(db) + .await; + + match batch { + Ok(log_files_to_delete) => { + if log_files_to_delete.is_empty() { + break; + } + let n = log_files_to_delete.len(); let paths = log_files_to_delete .iter() .map(|f| format!("{}/{}", f.hostname, f.file_path)) .collect(); - delete_log_files_from_disk_and_store(paths, &*TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await; - + delete_log_files_from_disk_and_store( + paths, + &*TMP_WINDMILL_LOGS_SERVICE, + windmill_common::tracing_init::LOGS_SERVICE, + ) + .await; + if (n as i64) < SERVICE_LOG_DELETE_BATCH { + break; + } + } + Err(e) => { + tracing::error!("Error deleting log file: {:?}", e); + break; + } } - Err(e) => tracing::error!("Error deleting log file: {:?}", e), } let audit_retention_days = audit_log_retention_days().await; @@ -2866,6 +2911,21 @@ pub async fn reload_retention_period_setting(conn: &Connection) { } } +pub async fn reload_service_log_retention_secs_setting(conn: &Connection) { + match load_setting_value::( + conn, + SERVICE_LOG_RETENTION_SECS_SETTING, + "SERVICE_LOG_RETENTION_SECS", + DEFAULT_SERVICE_LOG_RETENTION_SECS, + |x| x, + ) + .await + { + Ok(v) => windmill_common::set_service_log_retention_secs(v), + Err(e) => tracing::error!("Error reloading service log retention period: {:?}", e), + } +} + pub async fn reload_audit_log_retention_days_setting(conn: &Connection) { match load_setting_value::( conn, diff --git a/backend/windmill-api-settings/src/log_cleanup.rs b/backend/windmill-api-settings/src/log_cleanup.rs index ecbc38fcb7..82080cb61c 100644 --- a/backend/windmill-api-settings/src/log_cleanup.rs +++ b/backend/windmill-api-settings/src/log_cleanup.rs @@ -32,7 +32,7 @@ use windmill_common::tracing_init::{LOGS_SERVICE, TMP_WINDMILL_LOGS_SERVICE}; use windmill_common::worker::WINDMILL_DIR; use windmill_common::{ DB, INSTANCE_NAME, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, - JOB_RETENTION_SECS_OVERRIDES_LOADED, SERVICE_LOG_RETENTION_SECS, + JOB_RETENTION_SECS_OVERRIDES_LOADED, }; use windmill_object_store::object_store_reexports::{ @@ -249,7 +249,7 @@ async fn cleanup_service_logs( // Count candidates upfront for progress reporting. let total: i64 = sqlx::query_scalar!( "SELECT COUNT(*) FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval", - SERVICE_LOG_RETENTION_SECS, + windmill_common::service_log_retention_secs(), ) .fetch_one(db) .await? @@ -274,7 +274,7 @@ async fn cleanup_service_logs( WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval LIMIT $2 ) RETURNING file_path, hostname", - SERVICE_LOG_RETENTION_SECS, + windmill_common::service_log_retention_secs(), SERVICE_LOG_BATCH, ) .fetch_all(db) @@ -680,9 +680,10 @@ async fn cleanup_s3_orphans( ) -> error::Result<()> { let job_retention_secs = JOB_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed); let now = Utc::now(); - // Service logs always have a retention (hardcoded SERVICE_LOG_RETENTION_SECS), - // so we scan for service-log orphans regardless of JOB_RETENTION_SECS. - let service_cutoff = now - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS); + // Service logs always have a retention, so we scan for service-log orphans regardless of + // JOB_RETENTION_SECS. + let service_cutoff = + now - chrono::Duration::seconds(windmill_common::service_log_retention_secs()); // Job-log orphans are only considered once past a job's effective retention window. That window // is the instance one OR, for an override workspace (EE), its own — and jobs orphan their logs as diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index b539c1c6b3..b1ad2edd01 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -15,6 +15,7 @@ pub const OAUTH_SETTING: &str = "oauths"; pub const AI_CONFIG_SETTING: &str = "ai_config"; pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs"; pub const RETENTION_PERIOD_SECS_OVERRIDES_SETTING: &str = "retention_period_secs_overrides"; +pub const SERVICE_LOG_RETENTION_SECS_SETTING: &str = "service_log_retention_secs"; /// Upper bound on how many per-workspace retention overrides may be configured. The periodic monitor /// sweeps each override workspace in its own transaction every pass, so this keeps a pass bounded /// (and the feature is a targeted escape hatch for a handful of special workspaces, not a bulk knob). diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs index 7b1ed407ab..559eba3bf9 100644 --- a/backend/windmill-common/src/indexer.rs +++ b/backend/windmill-common/src/indexer.rs @@ -94,6 +94,21 @@ pub async fn load_indexer_config(db: &DB) -> error::Result i64 { + let retention = crate::service_log_retention_secs(); + if max_index_time_window_secs > 0 { + std::cmp::min(max_index_time_window_secs, retention) + } else { + retention + } +} + pub fn get_env_var(env_var: &str) -> Option { match std::env::var(env_var).map(|x| x.parse()) { Ok(Ok(i)) => Some(i), @@ -136,3 +151,60 @@ pub fn get_indexer_rates_from_env() -> TantivyIndexerSettings { settings } + +#[cfg(test)] +mod tests { + use super::*; + + // One test rather than several: both halves share the process-wide retention, and the + // setter half writes it, which parallel tests would race. + #[test] + fn retention_rejects_unusable_values_and_the_index_window_clamps_to_it() { + use crate::{ + service_log_retention_secs, set_service_log_retention_secs, + DEFAULT_SERVICE_LOG_RETENTION_SECS, + }; + + // See `set_service_log_retention_secs` for why the two unusable directions land apart: + // too large keeps the intent by capping, non-positive cannot and falls back. + let rejected: Vec = [0, -1, i64::MIN] + .iter() + .map(|v| { + set_service_log_retention_secs(*v); + service_log_retention_secs() + }) + .collect(); + let capped: Vec = [i64::MAX, 60 * 60 * 24 * 365 * 101] + .iter() + .map(|v| { + set_service_log_retention_secs(*v); + service_log_retention_secs() + }) + .collect(); + + set_service_log_retention_secs(60 * 60 * 24 * 3); + let retention = service_log_retention_secs(); + let windows = [ + // `0` disables the extra shrinking rather than lifting the ceiling — the trap that + // makes an unset setting look unbounded. + service_log_index_window_secs(0), + // Retention is the ceiling: the index cannot reach lines whose `log_file` row is gone. + service_log_index_window_secs(retention * 2), + service_log_index_window_secs(60), + ]; + set_service_log_retention_secs(DEFAULT_SERVICE_LOG_RETENTION_SECS); + + assert_eq!( + rejected, + vec![DEFAULT_SERVICE_LOG_RETENTION_SECS; 3], + "a value that would expire everything must fall back to the default" + ); + assert_eq!( + capped, + vec![60 * 60 * 24 * 365 * 100; 2], + "an oversized value must cap, not shorten retention to the default" + ); + assert_eq!(retention, 60 * 60 * 24 * 3); + assert_eq!(windows, [retention, retention, 60]); + } +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index cf0be2bbd8..f1784e9841 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -147,9 +147,53 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; -pub const SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs +pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers"; +/// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower +/// than an `i64`: `DateTime` subtraction panics past year 262143, and the `( s)::interval` +/// the cleanup queries build overflows Postgres' microsecond field. +const MAX_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100; + +/// Apply a configured service log retention, in seconds. +/// +/// The only way into [`SERVICE_LOG_RETENTION_SECS`], so an unusable value can never reach a +/// cutoff. The two unusable directions are not the same mistake and must not share a landing +/// point: too large still says "keep these for a very long time", so it is capped and the +/// intent survives, whereas falling back would delete logs the operator meant to keep. A +/// non-positive value has no such reading — every cutoff is `now - retention`, so it lands at +/// or after `now` and the next sweep expires the entire history, rows and object-storage files +/// alike. Unlike job retention there is no "keep forever" spelling here, so `0` — what an +/// operator types by analogy with it, and what the settings UI writes into a field that was +/// merely focused — falls back to the default. +pub fn set_service_log_retention_secs(configured: i64) { + let effective = if configured > MAX_SERVICE_LOG_RETENTION_SECS { + tracing::warn!( + "service log retention of {configured}s exceeds the maximum of \ + {MAX_SERVICE_LOG_RETENTION_SECS}s, capping it there" + ); + MAX_SERVICE_LOG_RETENTION_SECS + } else if configured >= 1 { + configured + } else { + tracing::warn!( + "service log retention of {configured}s would expire every service log, \ + falling back to the default of {DEFAULT_SERVICE_LOG_RETENTION_SECS}s" + ); + DEFAULT_SERVICE_LOG_RETENTION_SECS + }; + SERVICE_LOG_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); +} + +/// How long a service log line stays retrievable, in seconds. +/// +/// The outer bound on everything service-log: the `log_file` rows, the raw files in object +/// storage, the columnar store queried by retrieval, and — through +/// [`indexer::service_log_index_window_secs`] — the search index. +pub fn service_log_retention_secs() -> i64 { + SERVICE_LOG_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) +} + /// Canonical form of a base URL, used as one of the inputs to the offline-license /// instance hash (`compute_instance_hash`). /// @@ -375,6 +419,10 @@ lazy_static::lazy_static! { /// workspace configured before its override could be read. pub static ref JOB_RETENTION_SECS_OVERRIDES_LOADED: AtomicBool = AtomicBool::new(false); pub static ref AUDIT_LOG_RETENTION_DAYS: AtomicI64 = AtomicI64::new(0); + /// Private on purpose: [`set_service_log_retention_secs`] is the only writer, so a value that + /// would expire every service log cannot reach a cutoff. Read it with + /// [`service_log_retention_secs`]. + static ref SERVICE_LOG_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_SERVICE_LOG_RETENTION_SECS); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false); diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 91e00d8c71..4a0bd58715 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1143,6 +1143,32 @@ description="Configure default timeouts and retention policies for job execution." link="https://www.windmill.dev/docs/advanced/instance_settings#jobs" /> + {:else if category == 'Service logs'} + + {#if !$values['object_store_cache_config']} +
    + + Instance object storage is not configured, so every server and worker keeps its log + files on its own disk. This page lists what each host wrote, but can only open the + files belonging to the replica serving the request — another host's are listed and + not readable — and a host's files go with it when it is replaced. Retention below + still governs the entries in the database and the files on disk. + +
    + {:else if !$enterpriseLicense} +
    + + Log files are uploaded to instance object storage, and the indexer that would ingest + them into the columnar store and delete each one afterwards is an enterprise + feature. Retention below expires the database entries and the local files; the + uploaded copies are only removed when Delete logs from s3 periodically is on + under Object Storage. + +
    + {/if} {:else if category == 'Object Storage'} = { triggersRestart: true } ], + 'Service logs': [ + { + label: 'Retention in secs', + key: 'service_log_retention_secs', + description: + 'How long a service log is kept, across every copy of it: the entry in the database, the file on the disk of the process that wrote it, and — once instance object storage is configured and the indexer has ingested it — its line in the columnar store that search and the log viewer read. Search reaches back at most this far, and less when the indexer time window under Indexer is shorter. Defaults to 14 days. There is no keep-forever setting here — leave it empty for the default.', + fieldType: 'seconds', + storage: 'setting', + cloudonly: false, + error: + 'Service log retention must be between 1 second and 100 years — leave it empty for the default', + isValid: (value: any) => + value == undefined || + (typeof value === 'number' && value > 0 && value <= 60 * 60 * 24 * 365 * 100) + } + ], + Indexer: [ { label: '', @@ -1173,6 +1190,12 @@ export const instanceSettingsNavigationGroups = [ aiDescription: 'Instance OTEL/Prometheus settings', isEE: true }, + { + id: 'service_logs', + label: 'Service logs', + aiId: 'instance-settings-service-logs', + aiDescription: 'Service log retention settings' + }, { id: 'indexer', label: 'Indexer', @@ -1256,6 +1279,7 @@ export const tabToCategoryMap: Record = { webhooks: 'Webhooks', otel_prom: 'OTEL/Prom', indexer: 'Indexer', + service_logs: 'Service logs', telemetry: 'Telemetry', secret_storage: 'Secret Storage', object_storage: 'Object Storage', @@ -1291,6 +1315,7 @@ export const categoryToTabMap: Record = { Webhooks: 'webhooks', 'OTEL/Prom': 'otel_prom', Indexer: 'indexer', + 'Service logs': 'service_logs', Telemetry: 'telemetry', 'Secret Storage': 'secret_storage', 'Object Storage': 'object_storage', From 7639d83a4254fb5728ad1c7f200c6c197fcbbd4b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 29 Aug 2026 19:45:12 +0200 Subject: [PATCH 28/40] chore: bump ee-repo-ref to the merged EE main (#10896) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit windmill-ee-private#756 was squash-merged, so the commit ee-repo-ref names is not on EE main and the branch carrying it is gone. The content is identical, so nothing builds differently — but a dangling ref is one garbage collection away from an EE build that cannot fetch what it is pinned to. Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN Co-authored-by: Claude Opus 5 (1M context) --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 6f56f0ab9a..1ca9152918 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -e6483ff5a289521405912d95be5c5ba064bedb38 +f3c923975012e5e499fb07d8b390b61808bb8374 From d91ee4614a70f20a194f47e190327129f499ec63 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 30 Aug 2026 07:16:09 +0200 Subject: [PATCH 29/40] feat: day-partition the service log index and expire whole chunks (#10893) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: day-partition the service log index and expire whole chunks The service log index becomes one tantivy index per UTC day. The substance is in windmill-ee-private#753; this side carries the EE ref and moves the log indexer writer instead of cloning it, because sealing a chunk takes sole ownership of its tantivy writer. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX * fix: do not adopt the superseded watermark after an explicit index clear A clear asks for the retention window to be read again, and a watermark says it already has been — and the v3 copy in object storage is kept for rollback, so it outlives the local one the clear removes. Both copies of that watermark are now read and the newer wins, for the same reason the v4 one is taken from the store when it is ahead: a replica that lost the lock keeps a local file frozen where it stopped while the store went on. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX * fix: delete a day's raw files at its checkpoint, and rebuild whole days Bumps the EE ref for windmill-ee-private#753. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX * fix: make an interrupted rebuild detectable, and pin the rebuild floor Bumps the EE ref for windmill-ee-private#753. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX * fix: keep the rebuild marker in the object store, not on local disk Bumps the EE ref for windmill-ee-private#753. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX * fix: two more routes to a partial index being accepted as complete Bumps the EE ref for windmill-ee-private#753. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX * fix: trust a local chunk only when the tracker vouches for it Bumps the EE ref for windmill-ee-private#753. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX * chore: condense the stale-chunk guard's doc to the four-line limit Bumps the EE ref for windmill-ee-private#753. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX * chore: update ee-repo-ref to 17ef439b087b400889ff19109be9d2c810142278 This commit updates the EE repository reference after PR #753 was merged in windmill-ee-private. Previous ee-repo-ref: 3e79901b4742906d2285dd943e24fac0f735f199 New ee-repo-ref: 17ef439b087b400889ff19109be9d2c810142278 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/src/main.rs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 1ca9152918..ee579195bc 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -f3c923975012e5e499fb07d8b390b61808bb8374 +17ef439b087b400889ff19109be9d2c810142278 diff --git a/backend/src/main.rs b/backend/src/main.rs index c782e1ad97..fa88cf6cdc 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1264,10 +1264,12 @@ Windmill Community Edition {GIT_VERSION} #[cfg(all(feature = "tantivy", feature = "parquet"))] let log_indexer_f = { let log_indexer_rx = killpill_rx.resubscribe(); - let log_index_writer2 = log_index_writer.clone(); + // Moved, not cloned: sealing a chunk takes sole ownership of its + // tantivy writer, which a second live handle would silently prevent. + let moved_log_index_writer = log_index_writer; async { if let Some(db) = conn.as_sql() { - if let Some(log_index_writer) = log_index_writer2 { + if let Some(log_index_writer) = moved_log_index_writer { windmill_indexer::service_logs_oss::run_indexer( db.clone(), log_index_writer, From ac56586c0e56d4022761d3c80306a03d57f8bfcb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 30 Aug 2026 07:40:34 +0200 Subject: [PATCH 30/40] fix: correct the service log ingest flush boundary (#10898) Claude-Session: https://claude.ai/code/session_014KnE8my2okxQGCx47cWMjf Co-authored-by: Claude Opus 5 (1M context) --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index ee579195bc..5613e1173c 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -17ef439b087b400889ff19109be9d2c810142278 +25d911019aebd3779bb9461a5faab4c20e2f9cf0 From 2fb790338d130c9ce64afe70c8f9fd0fd091219a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 31 Aug 2026 08:48:27 +0200 Subject: [PATCH 31/40] upgrade argon2 to 0.6 and migrate the password hashing API (#10902) * fix: upgrade argon2 to 0.6 and migrate the password hashing API * test: pin that an unparseable stored hash reads as a failed login * chore: update ee-repo-ref to 58738c39ac41d57917bbd9400318704763d997f7 This commit updates the EE repository reference after PR #759 was merged in windmill-ee-private. Previous ee-repo-ref: 02a89fc4d27e49a494112fa91a8812e3ee4fb8a6 New ee-repo-ref: 58738c39ac41d57917bbd9400318704763d997f7 Automated by sync-ee-ref workflow. --------- Co-authored-by: windmill-internal-app[bot] --- backend/Cargo.lock | 100 +++++++++++++++--- backend/Cargo.toml | 8 +- backend/ee-repo-ref.txt | 2 +- .../tests/users.rs | 23 ++-- backend/windmill-api-users/src/users.rs | 26 ++++- 5 files changed, 128 insertions(+), 31 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7c8d40ad05..9bfefb16d9 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -23,7 +23,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -261,13 +261,13 @@ dependencies = [ [[package]] name = "argon2" -version = "0.5.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c" dependencies = [ "base64ct", - "blake2", - "cpufeatures 0.2.17", + "blake2 0.11.0", + "cpufeatures 0.3.1", "password-hash", ] @@ -1824,6 +1824,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "blake2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "blake3" version = "1.8.7" @@ -1856,6 +1865,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-modes" version = "0.8.1" @@ -2390,7 +2408,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -2454,6 +2472,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cms" version = "0.2.3" @@ -2818,6 +2842,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -2848,6 +2881,15 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curl-sys" version = "0.4.90+curl-8.21.0" @@ -3413,7 +3455,7 @@ dependencies = [ "arrow", "arrow-buffer", "base64 0.22.1", - "blake2", + "blake2 0.10.6", "blake3", "chrono", "datafusion-common", @@ -4455,10 +4497,21 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dirs" version = "4.0.0" @@ -6139,6 +6192,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -8924,13 +8986,12 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.5.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", + "getrandom 0.4.3", + "phc", ] [[package]] @@ -9074,6 +9135,17 @@ dependencies = [ "phf 0.11.3", ] +[[package]] +name = "phc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +dependencies = [ + "base64ct", + "ctutils", + "getrandom 0.4.3", +] + [[package]] name = "phf" version = "0.11.3" @@ -14145,7 +14217,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 3f614c9388..86940805fc 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -477,10 +477,10 @@ rust-embed = { version = "^6", features = ["interpolate-folder-path"] } mime_guess = "^2" hex = "^0" sql-builder = "^3" -# Pinned: `^0` floats to 0.6, which moved `password_hash::SaltString`, put -# `rand_core` behind a feature and changed `hash_password`'s signature — -# users_ee.rs is written against 0.5 and does not compile otherwise. -argon2 = "0.5" +# Minor-pinned rather than the `^0` used elsewhere in this file: argon2's 0.x +# minors are API-breaking (0.6 moved `SaltString` into `phc`, put `rand_core` +# behind a feature and changed `hash_password`), so a float breaks the build. +argon2 = "0.6" quick_cache = "^0" rand = "=0.9.0" rand_core = { version = "^0", features = ["std"] } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 5613e1173c..8e49c42d98 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -25d911019aebd3779bb9461a5faab4c20e2f9cf0 +58738c39ac41d57917bbd9400318704763d997f7 diff --git a/backend/windmill-api-integration-tests/tests/users.rs b/backend/windmill-api-integration-tests/tests/users.rs index fb64d9eb77..bec59ac45a 100644 --- a/backend/windmill-api-integration-tests/tests/users.rs +++ b/backend/windmill-api-integration-tests/tests/users.rs @@ -308,14 +308,17 @@ async fn test_user_endpoints(db: Pool) -> anyhow::Result<()> { let auth_base = format!("http://localhost:{port}/api/auth"); // --- login (will fail: password hash in fixture is fake) --- + // An unparseable stored hash must read as a failed login, not as a server error + // relaying the hash parser's message to an unauthenticated caller. let resp = client() .post(format!("{auth_base}/login")) .json(&json!({"email": "test@windmill.dev", "password": "wrong-password"})) .send() .await .unwrap(); - assert!( - resp.status() == 400 || resp.status() == 401 || resp.status() == 500, + assert_eq!( + resp.status(), + 400, "login: unexpected status {}", resp.status() ); @@ -804,12 +807,16 @@ async fn test_change_user_email_leaves_group_identities(db: Pool) -> a let server = ApiServer::start(db.clone()).await?; let global_base = format!("http://localhost:{}/api/users", server.addr.port()); - sqlx::query!("UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'") - .execute(&db) - .await?; - sqlx::query!("UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'") - .execute(&db) - .await?; + sqlx::query!( + "UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'" + ) + .execute(&db) + .await?; + sqlx::query!( + "UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'" + ) + .execute(&db) + .await?; sqlx::query!( "INSERT INTO group_(workspace_id, name, summary, extra_perms) VALUES ('test-workspace', 'ops', '', '{}')" ) diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 8ea3bd0995..8daa414b46 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -19,7 +19,7 @@ use windmill_api_auth::ApiAuthed; pub use windmill_api_auth::Tokened; -use argon2::{Argon2, PasswordHash, PasswordVerifier}; +use argon2::{Argon2, PasswordVerifier}; use axum::{ extract::{Extension, Path, Query}, response::{IntoResponse, Response}, @@ -2680,10 +2680,8 @@ async fn login( .await?; if let Some((email, hash, super_admin)) = email_w_h { - let parsed_hash = - PasswordHash::new(&hash).map_err(|e| Error::internal_err(e.to_string()))?; if argon2 - .verify_password(password.as_bytes(), &parsed_hash) + .verify_password(password.as_bytes(), hash.as_str()) .is_err() { audit_log( @@ -3710,3 +3708,23 @@ async fn request_password_reset( } // NOTE: reset_password is in windmill-api (depends on users_oss::hash_password EE dispatch) + +#[cfg(test)] +mod tests { + use super::*; + + /// Stored hashes outlive the hashing crate: every instance still holds hashes minted by + /// older argon2 releases, and an upgrade that stopped reading them locks their users out. + #[test] + fn verifies_a_hash_minted_by_an_older_argon2() { + // The seeded admin hash from migration 20220508150023, m=4096,t=3,p=1. + let seeded = "$argon2id$v=19$m=4096,t=3,p=1$oLJo/lPn/gezXCuFOEyaNw$i0T2tCkw3xUFsrBIKZwr8jVNHlIfoxQe+HfDnLtd12I"; + + assert!(Argon2::default() + .verify_password(b"changeme", seeded) + .is_ok()); + assert!(Argon2::default() + .verify_password(b"not-the-password", seeded) + .is_err()); + } +} From aa4a6ffd66813010a79c07741b01a984ed4e7df6 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 31 Aug 2026 14:06:29 +0200 Subject: [PATCH 32/40] fix: track outstanding service log files on the rows themselves (#10894) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: track outstanding service log files on the rows themselves Adds `log_file.indexed_at` so the service log ingest can read outstanding rows instead of walking a cursor over `log_ts`. A row registered after the pass had gone by its minute was skipped for good, and no ordering fixes that — an arrival sequence fails the same way, since a row can take a lower value and commit after a higher one has moved the cursor past it. The migration marks existing rows with a sentinel; the first pass returns the ones the old cursor had not reached to the queue. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EPAP96jJNYpPQ8bpxZcU1C * [ee] refactor: drop the claim/confirm phase from the service log ingest queue Two states are enough: a row is outstanding or it is marked. The migration no longer creates the index for the claim sentinel, and the sqlx cache loses the two queries the event-time cursor used. * [ee] fix: make re-indexing a service log file idempotent Corrects the `init_last_log_file_sent` note: a rewritten row keeps the `indexed_at` it had, so one the indexers already took is not offered again. * [ee] fix: let a rebuild take the rows it covered out of the ingest queue Adds the query that releases them; the index layout stays v4. * [ee] fix: index the lookup a rebuild releases rows by A rebuild takes rows out of the queue by the file it read out of the store, which is the one lookup that arrives without a `log_ts`. The primary key is `(hostname, log_ts)`, so nothing covered it and each batch scanned every outstanding row — worst in exactly the state a rebuild follows. Verified at 50k outstanding rows: sequential scan becomes an index scan. Also records `log_file.indexed_at` in the schema reference. * [ee] fix: treat a state handed back without its line count as behind * [ee] fix: give the converted state a line count * [ee] fix: keep the converted cursor from being rewound by the rebuild * [ee] fix: inherit the legacy cursor from one source, not field by field * [ee] fix: count a file's lines against the buffer before reading it * [ee] fix: bound the row buffer on what it holds, not on reported counts * [ee] fix: settle the upgrade from the store rather than from event time * [ee] docs: describe the conversion's second half as it now works * [ee] refactor: settle the upgrade with one rebuild instead of reconciling The migration records existing rows as done rather than marking them with a sentinel: the indexer puts back what the old cursor had not reached on its first pass, which is the only place that cursor's position is known. * [ee] fix: repair the rows the old cursor skipped instead of recording them as done The migration marks pre-existing rows with a sentinel again, so the indexer can tell them from rows registered since and put the window's worth back on the queue. * [ee] fix: keep a source file whole in one partition * [ee] revert the file-atomic partition change * [ee] fix: dedupe the public reads, and repair an index without a cursor * [ee] fix: repair an index whose cursor is gone, and keep what the repair found * [ee] fix: seed a pass from both axes of what a rebuild recovered * [ee] fix: settle the cursor on what the store holds, not on what was read * [ee] fix: an empty rebuild must not claim ground it has not covered * [ee] test: pin the cursor a rebuild settles on * chore: update ee-repo-ref to bc0c7051585194474078b6c1941a3fb73893d9e5 This commit updates the EE repository reference after PR #755 was merged in windmill-ee-private. Previous ee-repo-ref: 328f5a90afeae9c683bf3294f0d9eb293a3e1a92 New ee-repo-ref: bc0c7051585194474078b6c1941a3fb73893d9e5 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- AGENTS.md | 9 +++ ...463a0e7b5248c226ee92d95df94c3099cc400.json | 15 +++++ ...b14b842f595f68dd885248abaaeabd2d0bf1.json} | 5 +- ...4633c1531170ded1fdf09114718b941f5e1db.json | 65 ------------------- ...cdc85389c0629847d8fabb2ad0aa043957b2f.json | 22 +++++++ ...0c06924719189bec8e5c19866db8d0b87df5e.json | 15 +++++ ...fd56e616ee79d895b7dcc37ad9442789e1574.json | 22 +++++++ ...53d23ec2af084b2f93da24c920532c1916384.json | 6 +- backend/ee-repo-ref.txt | 2 +- ...0260830085453_log_file_indexed_at.down.sql | 4 ++ .../20260830085453_log_file_indexed_at.up.sql | 35 ++++++++++ backend/src/monitor.rs | 13 ++-- backend/summarized_schema.txt | 2 +- 13 files changed, 136 insertions(+), 79 deletions(-) create mode 100644 backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json rename backend/.sqlx/{query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json => query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json} (86%) delete mode 100644 backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json create mode 100644 backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json create mode 100644 backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json create mode 100644 backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json create mode 100644 backend/migrations/20260830085453_log_file_indexed_at.down.sql create mode 100644 backend/migrations/20260830085453_log_file_indexed_at.up.sql diff --git a/AGENTS.md b/AGENTS.md index 47919cfeda..71c59b479d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -165,6 +165,15 @@ $NAV --root backend callees "X" # what does X call? - Search for existing code to reuse before writing new code - Follow established patterns in the codebase - Keep changes focused — don't refactor beyond what's asked +- **A simpler design found late is still the design.** Work already spent is not an argument + for a shape, and neither is a clean review round, a passing suite, or a long PR thread. The + signal to stop and re-derive rather than patch again is a change that keeps growing to defend + its own structure: each review finding fixing an assumption the previous fix broke, the same + class of bug reappearing somewhere new, or most of the diff being consequences of one early + choice rather than the thing you set out to do. When that happens, say plainly what the + simpler design is and what switching costs — a migration, a review cycle restarted from zero, + work discarded — and let the user decide. Do not keep paying down the harder one because it + is nearly finished, and do not present the accumulated cost as a reason to continue. - **Ship only the tests the PR needs.** A committed test must pin behavior a future change could plausibly break, and be the smallest setup that exercises the new logic. While developing, write as many exhaustive tests and do as much manual testing as you need to convince yourself the change works — then remove that scaffolding before marking the PR ready, keeping only the essential regression guard(s). A test that merely re-exercises pre-existing behavior, or needs elaborate fixtures to assert something trivial, is scaffolding: delete it. If nothing meaningful is left to guard, ship no test rather than a ceremonial one. - **Comments record constraints, not narration.** Write a comment only for what the code can't show: why a non-obvious approach is required, what breaks if it's "simplified" away. State each invariant once, at the place where someone would break it, in ≤4 lines. Don't describe what the next line does, don't repeat the same rationale at multiple sites, and don't address the PR reviewer (justifying a change belongs in the PR description, not the code). Reference nothing ephemeral — no numbered steps from your dev flow, no "the poller / the test does X" scaffolding, no transient state that won't exist for the next reader; keep only the essential, durable rationale. Describe the code as it is, never its drafting history: "we no longer do X", "unchanged behavior", "instead of the previous approach" are meaningless to a reader who never saw the earlier iteration — before finishing, reread your comments as if the current state is the only state that ever existed. - **Never attribute work to a specific customer, account, or "requested by a customer" in repo-tracked content** (PR descriptions, commit messages, code comments, docs). Describe changes by their technical motivation instead. diff --git a/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json b/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json new file mode 100644 index 0000000000..3713af1486 --- /dev/null +++ b/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::text[]) AS c(hostname, file_path)\n WHERE log_file.indexed_at IS NULL\n AND log_file.hostname = c.hostname\n AND log_file.file_path = c.file_path", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400" +} diff --git a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json b/backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json similarity index 86% rename from backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json rename to backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json index ba9f3814e8..13d1c9b1e8 100644 --- a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json +++ b/backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > $1\n ORDER BY log_ts ASC LIMIT $2", + "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE indexed_at IS NULL\n ORDER BY log_ts ASC, hostname ASC LIMIT $1", "describe": { "columns": [ { @@ -46,7 +46,6 @@ ], "parameters": { "Left": [ - "Timestamp", "Int8" ] }, @@ -61,5 +60,5 @@ true ] }, - "hash": "b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a" + "hash": "6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1" } diff --git a/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json b/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json deleted file mode 100644 index d0b96444ee..0000000000 --- a/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > NOW() - make_interval(secs => $1)\n ORDER BY log_ts ASC LIMIT $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "hostname", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "mode", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "worker_group", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "log_ts", - "type_info": "Timestamp" - }, - { - "ordinal": 4, - "name": "file_path", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "ok_lines", - "type_info": "Int8" - }, - { - "ordinal": 6, - "name": "err_lines", - "type_info": "Int8" - }, - { - "ordinal": 7, - "name": "json_fmt", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Float8", - "Int8" - ] - }, - "nullable": [ - false, - null, - true, - false, - false, - true, - true, - true - ] - }, - "hash": "8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db" -} diff --git a/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json b/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json new file mode 100644 index 0000000000..96745f9485 --- /dev/null +++ b/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH moved AS (\n UPDATE log_file SET indexed_at = CASE\n WHEN log_ts > NOW() - make_interval(secs => $1) THEN NULL\n ELSE now() END\n WHERE indexed_at = 'epoch' RETURNING 1)\n SELECT count(*) AS \"n!\" FROM moved", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f" +} diff --git a/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json b/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json new file mode 100644 index 0000000000..6739d80033 --- /dev/null +++ b/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::timestamp[]) AS c(hostname, log_ts)\n WHERE log_file.hostname = c.hostname AND log_file.log_ts = c.log_ts", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TimestampArray" + ] + }, + "nullable": [] + }, + "hash": "947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e" +} diff --git a/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json b/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json new file mode 100644 index 0000000000..a83c34e0db --- /dev/null +++ b/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH retired AS (\n UPDATE log_file SET indexed_at = now()\n WHERE indexed_at IS NULL\n AND log_ts <= NOW() - make_interval(secs => $1) RETURNING 1)\n SELECT count(*) AS \"n!\" FROM retired", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574" +} diff --git a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json index b336210daf..b892061f56 100644 --- a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json +++ b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json @@ -98,12 +98,12 @@ null, null, null, - true, + false, null, null, null, - true, - true + false, + false ] }, "hash": "b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 8e49c42d98..693921efc6 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -58738c39ac41d57917bbd9400318704763d997f7 +bc0c7051585194474078b6c1941a3fb73893d9e5 diff --git a/backend/migrations/20260830085453_log_file_indexed_at.down.sql b/backend/migrations/20260830085453_log_file_indexed_at.down.sql new file mode 100644 index 0000000000..ac3c92f3eb --- /dev/null +++ b/backend/migrations/20260830085453_log_file_indexed_at.down.sql @@ -0,0 +1,4 @@ +DROP INDEX IF EXISTS index_log_file_premigration; +DROP INDEX IF EXISTS index_log_file_pending_path; +DROP INDEX IF EXISTS index_log_file_pending; +ALTER TABLE log_file DROP COLUMN IF EXISTS indexed_at; diff --git a/backend/migrations/20260830085453_log_file_indexed_at.up.sql b/backend/migrations/20260830085453_log_file_indexed_at.up.sql new file mode 100644 index 0000000000..3421a5082a --- /dev/null +++ b/backend/migrations/20260830085453_log_file_indexed_at.up.sql @@ -0,0 +1,35 @@ +-- The service log ingest walked `log_file` with a cursor over `log_ts`, which is when a line +-- was written rather than when its row appeared. Rows do not arrive in that order — an upload +-- retried after a failure, a host that has just started, a batch the row limit cut mid-minute — +-- and a row that becomes visible behind the cursor is never read: it stays in `log_file` and its +-- lines stay out of search until retention drops them. +-- +-- No ordering fixes this. A cursor over arrival order fails the same way, because `nextval` is +-- allocated before its INSERT commits: a row can be assigned a lower value and commit after a +-- higher one has already moved the cursor past it. Which rows are outstanding is a property of +-- the rows, so it is recorded on them. +ALTER TABLE log_file ADD COLUMN indexed_at TIMESTAMPTZ; + +-- Rows that already existed are marked, not queued: on a 14-day window most were ingested long +-- ago and their raw files are gone. A sentinel rather than a timestamp, because the indexer has to +-- tell them apart from rows registered since — those start NULL — and it puts the window's worth of +-- them back on the queue on its first pass, keeping only what the columnar store can vouch for. +-- +-- Not split here on the cursor the old ingest had reached. Below that cursor sits every row it +-- skipped, which is the loss this migration exists to stop; recording those as done would carry the +-- bug into its own fix. +UPDATE log_file SET indexed_at = 'epoch' WHERE indexed_at IS NULL; + +-- The work queue, and the only index the ingest query needs: outstanding rows are a small +-- fraction of the table, so this stays proportional to what is left to do rather than to the +-- retention window. +CREATE INDEX index_log_file_pending ON log_file (log_ts) WHERE indexed_at IS NULL; + +-- A rebuild takes rows out of the queue by the file it just read out of the store, which is +-- the one lookup that arrives without a `log_ts`: the primary key is `(hostname, log_ts)`, so +-- nothing else covers it and each batch would scan every outstanding row instead. +CREATE INDEX index_log_file_pending_path ON log_file (hostname, file_path) WHERE indexed_at IS NULL; + +-- Reached once per pass while pre-migration rows survive, and never again after the first +-- conversion clears them. +CREATE INDEX index_log_file_premigration ON log_file (log_ts) WHERE indexed_at = 'epoch'; diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index c0748a3798..ce09cebe71 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1358,9 +1358,9 @@ fn last_log_file_sent() -> Option { /// the file that was still open and the appender reopens that minute in append mode, /// so a restart inside it would otherwise strand everything written afterwards. /// -/// A row rewritten this way restores the object and sums the counters, but whether the -/// indexers read it again depends on their single `log_ts >` cursor, which is not -/// per-hostname: a minute at or below it stays out of search until it is re-indexed. +/// A row rewritten this way restores the object and sums the counters, but it keeps the +/// `indexed_at` it already had, so one the indexers have taken is not offered again and +/// the lines added by the rewrite stay out of search. async fn init_last_log_file_sent(conn: &Connection, hostname: &str) { let Some(db) = conn.as_sql() else { return; @@ -1406,9 +1406,10 @@ async fn send_log_files_to_object_store( if ts < retention_cutoff { continue; } - // Stop at the first failure rather than moving on: both indexers walk - // `log_file` with a `log_ts > watermark` cursor, so a row that lands after - // a newer one is never picked up. + // Stop at the first failure rather than moving on, so a file is never + // registered before an older one that has not made it to the store yet. + // The indexers do not depend on that ordering — every row is offered until + // it is marked — but a gap here would still be visible while it lasts. if !send_log_file_to_object_store(hostname, mode, worker_group, conn, &file_name, ts).await { break; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index a9865cebd3..607ef99e0c 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -128,7 +128,7 @@ job_stats: workspace_id(char), job_id(uuid), metric_id(char), metric_name(char), kafka_pending_commits: id(bigint), workspace_id(char), kafka_trigger_path(char), topic(char), partition(int), offset(bigint), created_at(ts) FK: (workspace_id, kafka_trigger_path) -> kafka_trigger(workspace_id, path) kafka_trigger: path(char), kafka_resource_path(char), topics(char), group_id(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), filters(jsonb[]), auto_commit(bool), labels(text[]) -log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool) +log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool), indexed_at(ts) macro_definition: workspace_id(char), name(char), provider_path(char), params(text), body(text), is_table_macro(bool), created_at(ts) FK: (workspace_id) -> workspace(id) macro_usage: workspace_id(char), consumer_path(char), macro_name(char) From 66123f3a9b8978c0084b02f50b44cffba125a13a Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:26:08 +0200 Subject: [PATCH 33/40] feat: add --keep-deleted flag to wmill sync pull and push (#10878) * feat: add --keep-deleted flag to wmill sync pull and push Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JMfhFSZQJJRLPrfkug6VoK * fix: address review findings on --keep-deleted Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JMfhFSZQJJRLPrfkug6VoK --------- Co-authored-by: Claude Opus 5 (1M context) --- cli/src/commands/shared_ui.ts | 152 +++++++++++++----- cli/src/commands/sync/sync.ts | 94 +++++++++-- cli/src/guidance/skills.gen.ts | 2 + cli/test/shared_ui_diff_unit.test.ts | 61 ++++++- cli/test/sync_pull_push.test.ts | 99 ++++++++++++ .../auto-generated/cli/cli-commands.md | 2 + system_prompts/auto-generated/prompts.ts | 2 + .../skills/cli-commands/SKILL.md | 2 + 8 files changed, 353 insertions(+), 61 deletions(-) diff --git a/cli/src/commands/shared_ui.ts b/cli/src/commands/shared_ui.ts index 3c01870fd1..f8e2965994 100644 --- a/cli/src/commands/shared_ui.ts +++ b/cli/src/commands/shared_ui.ts @@ -26,37 +26,33 @@ async function readDirRecursive( return out; } +/** The workspace's shared UI store, or undefined when it cannot be read. */ +async function fetchSharedUi( + workspace: string, +): Promise | undefined> { + try { + const got = await wmill.getSharedUi({ workspace }); + return got.files ?? {}; + } catch { + return undefined; + } +} + export type SharedUiChange = | { type: "added"; path: string } | { type: "edited"; path: string; before: string; after: string } | { type: "deleted"; path: string }; /** - * Diff the local /ui/ folder against the workspace's shared UI store in - * the push direction (local -> remote), returning entries whose `path` is - * prefixed with `ui/`. This is the same comparison pushSharedUi applies, so the - * dry-run preview and the real push never diverge. - * - * Mirrors pushSharedUi's no-op: with no local ui/ folder there is nothing to - * push, so the apply is a no-op and the preview must be empty (even when the - * remote store is non-empty) to avoid phantom diffs the apply won't perform. + * The push-direction diff (local -> remote) of two already-read file maps. + * Split out so the preview and the push itself compare identically off one + * read of each side. */ -export async function diffSharedUi(workspace: string): Promise { - const localDir = path.join(process.cwd(), SHARED_UI_DIR); - if (!fs.existsSync(localDir)) { - return []; - } - const files = await readDirRecursive(localDir); - - let remote: Record = {}; - try { - const got = await wmill.getSharedUi({ workspace }); - remote = got.files ?? {}; - } catch { - // If endpoint missing or unauthorized, treat remote as empty (the push - // would attempt the PUT anyway). - } - +function sharedUiChanges( + files: Record, + remote: Record, + keepDeleted?: boolean, +): SharedUiChange[] { // Use Object.hasOwn, not `in`: a file named after an Object.prototype member // (e.g. ui/toString) would otherwise register as always-present and be // misdiffed. @@ -69,34 +65,97 @@ export async function diffSharedUi(workspace: string): Promise changes.push({ type: "edited", path: p, before: remote[rel], after: content }); } } - for (const rel of Object.keys(remote)) { - if (!Object.hasOwn(files, rel)) { - changes.push({ type: "deleted", path: `${SHARED_UI_DIR}/${rel}` }); + if (!keepDeleted) { + for (const rel of Object.keys(remote)) { + if (!Object.hasOwn(files, rel)) { + changes.push({ type: "deleted", path: `${SHARED_UI_DIR}/${rel}` }); + } } } return changes; } +/** + * Diff the local /ui/ folder against the workspace's shared UI store in + * the push direction (local -> remote), returning entries whose `path` is + * prefixed with `ui/`. This is the same comparison pushSharedUi applies, so the + * dry-run preview and the real push never diverge. + * + * Mirrors pushSharedUi's no-ops: with no local ui/ folder, or under + * `keepDeleted` with an unreadable store, the apply does nothing, so the + * preview must be empty (even when the remote store is non-empty) to avoid + * phantom diffs the apply won't perform. + */ +export async function diffSharedUi( + workspace: string, + keepDeleted?: boolean, +): Promise { + const localDir = path.join(process.cwd(), SHARED_UI_DIR); + if (!fs.existsSync(localDir)) { + return []; + } + const files = await readDirRecursive(localDir); + const remote = await fetchSharedUi(workspace); + if (remote === undefined && keepDeleted) { + return []; + } + // If endpoint missing or unauthorized, treat remote as empty (the push + // would attempt the PUT anyway). + return sharedUiChanges(files, remote ?? {}, keepDeleted); +} + /** * Push the local /ui/ folder to the workspace's shared UI store. * Returns true if a push was performed, false if the folder is missing or * already matches the remote store. Note an empty-but-existing folder still - * pushes an empty map (clearing the remote store) if the remote is non-empty. + * pushes an empty map (clearing the remote store) if the remote is non-empty — + * unless `keepDeleted`, which folds remote-only files back into the map. */ -export async function pushSharedUi(workspace: string): Promise { +export async function pushSharedUi( + workspace: string, + keepDeleted?: boolean, +): Promise { const localDir = path.join(process.cwd(), SHARED_UI_DIR); if (!fs.existsSync(localDir)) { return false; } - // Skip if no change — reuse diffSharedUi so preview and push never diverge. - const diff = await diffSharedUi(workspace); - if (diff.length === 0) { + const files = await readDirRecursive(localDir); + const remote = await fetchSharedUi(workspace); + // The store is written whole, so a remote-only file is pruned by omission + // alone. An unreadable store leaves no way to carry those files over, so + // the shared UI is left untouched rather than cleared. + if (remote === undefined && keepDeleted) { + log.warn( + colors.yellow( + "Could not read the shared UI folder from the remote; skipping its push so --keep-deleted does not clear it.", + ), + ); + return false; + } + + // Same comparison as diffSharedUi, off the same two maps, so preview and + // push never diverge. + if (sharedUiChanges(files, remote ?? {}, keepDeleted).length === 0) { log.info(colors.gray("Shared UI folder up to date")); return false; } - const files = await readDirRecursive(localDir); + if (keepDeleted) { + for (const [rel, content] of Object.entries(remote!)) { + // defineProperty, not assignment: `files.__proto__ = "..."` runs the + // inherited setter and creates no own property, so a remote `ui/__proto__` + // would be dropped from the whole-map PUT — i.e. deleted despite the flag. + if (!Object.hasOwn(files, rel)) { + Object.defineProperty(files, rel, { + value: content, + writable: true, + enumerable: true, + configurable: true, + }); + } + } + } await wmill.updateSharedUi({ workspace, requestBody: { files }, @@ -111,9 +170,12 @@ export async function pushSharedUi(workspace: string): Promise { /** * Pull the workspace's shared UI store into /ui/. - * Files removed remotely are also removed locally. + * Files removed remotely are also removed locally, unless `keepDeleted`. */ -export async function pullSharedUi(workspace: string): Promise { +export async function pullSharedUi( + workspace: string, + keepDeleted?: boolean, +): Promise { const localDir = path.join(process.cwd(), SHARED_UI_DIR); let got; try { @@ -145,15 +207,17 @@ export async function pullSharedUi(workspace: string): Promise { } // Delete locally-orphaned files - const known = new Set(Object.keys(files)); - const local = await readDirRecursive(localDir); - for (const rel of Object.keys(local)) { - if (!known.has(rel)) { - const full = path.join(localDir, rel); - try { - fs.unlinkSync(full); - } catch { - // ignore + if (!keepDeleted) { + const known = new Set(Object.keys(files)); + const local = await readDirRecursive(localDir); + for (const rel of Object.keys(local)) { + if (!known.has(rel)) { + const full = path.join(localDir, rel); + try { + fs.unlinkSync(full); + } catch { + // ignore + } } } } diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index b1fca78ed2..cf44d5ec80 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -3174,6 +3174,28 @@ export function untrackedDatatableMigrationDeletions< ); } +/** + * `--keep-deleted`: strip every deletion from the changeset, in place, so the + * sync only adds and updates. A path missing on one side is not on its own + * evidence that it should go from the other — a partial clone, a scoped + * checkout or an item authored in the UI all read as deletions here. + */ +function dropDeletions(changes: Change[], keptOn: "local" | "remote"): void { + const deletions = changes.filter((c) => c.name === "deleted"); + if (deletions.length === 0) return; + const kept = changes.filter((c) => c.name !== "deleted"); + changes.length = 0; + changes.push(...kept); + log.info( + colors.yellow( + `--keep-deleted: keeping ${deletions.length} item(s) that exist only ` + + (keptOn === "local" + ? `on disk instead of deleting them locally` + : `on the remote instead of deleting them from the workspace`), + ), + ); +} + interface ChangeTracker { scripts: string[]; flows: string[]; @@ -3433,6 +3455,7 @@ export async function pull( repository?: string; promotion?: string; branch?: string; + keepDeleted?: boolean; useIndividualBranch?: boolean; groupByFolder?: boolean; gitDeployItems?: string; @@ -3684,6 +3707,10 @@ export async function pull( await isCaseInsensitiveFilesystem(process.cwd()), ); + if (opts.keepDeleted) { + dropDeletions(changes, "local"); + } + log.info( `remote (${workspace.name}) -> local: ${changes.length} changes to apply`, ); @@ -4117,10 +4144,14 @@ export async function pull( } } - try { - await pullSharedUi(workspace.workspaceId); - } catch (e) { - log.warn(`Failed to pull shared UI folder: ${e}`); + // Skipped under --dry-run since pullSharedUi writes to the local ui/ folder. + // An empty changeset falls through the return above and reaches here. + if (!opts.dryRun) { + try { + await pullSharedUi(workspace.workspaceId, opts.keepDeleted); + } catch (e) { + log.warn(`Failed to pull shared UI folder: ${e}`); + } } // Datatable migrations are part of the workspace export now, so they flow @@ -4425,12 +4456,19 @@ function removeSuffix(str: string, suffix: string) { } // Shown after a `wmill sync push --dry-run` preview that has changes. `sync push` -// deploys to the remote workspace and is destructive (it overwrites and prunes -// remote items that differ from or are absent locally), so the preview reminds -// the caller — especially an AI agent that ran the dry-run to inspect changes — -// to get explicit user confirmation before applying it for real. -const SYNC_PUSH_DESTRUCTIVE_WARNING = - "`wmill sync push` is destructive: applying it deploys these changes to the remote workspace and overwrites or deletes remote items that differ from or are absent locally — this is not automatically reversible. If you are an AI agent, do NOT run `wmill sync push` (without --dry-run) until the user has explicitly confirmed this deploy, unless your custom instructions explicitly allow bypassing that confirmation."; +// deploys to the remote workspace and is destructive (it overwrites remote items +// that differ from local, and prunes those absent locally unless --keep-deleted), +// so the preview reminds the caller — especially an AI agent that ran the dry-run +// to inspect changes — to get explicit user confirmation before applying it for real. +function syncPushDestructiveWarning(keepDeleted?: boolean): string { + return ( + "`wmill sync push` is destructive: applying it deploys these changes to the remote workspace and overwrites " + + (keepDeleted + ? "remote items that differ from local" + : "or deletes remote items that differ from or are absent locally") + + " — this is not automatically reversible. If you are an AI agent, do NOT run `wmill sync push` (without --dry-run) until the user has explicitly confirmed this deploy, unless your custom instructions explicitly allow bypassing that confirmation." + ); +} // A script pushed without a local lock queues a server-side dependency job; if // that job fails the script deploys broken (no lock/assets) with no CLI signal. @@ -4490,6 +4528,7 @@ export async function push( SyncOptions & { repository?: string; branch?: string; + keepDeleted?: boolean; acceptOverridingPermissionedAsWithSelf?: boolean; }, ) { @@ -4791,6 +4830,13 @@ export async function push( ); } + // After the shared-lock pass, which reads a shared lockfile's deletion as the + // signal that this checkout is not deduplicated — an advisory about the local + // tree that holds whether or not remote items are being kept. + if (opts.keepDeleted) { + dropDeletions(changes, "remote"); + } + const autoRegenerate = !!(opts as any).autoMetadata; const staleScripts: string[] = []; const staleFlows: string[] = []; @@ -5103,7 +5149,10 @@ export async function push( // unchanged (pushSharedUi still runs) and the summary count includes ui/. if (opts.dryRun) { try { - for (const c of await diffSharedUi(workspace.workspaceId)) { + for (const c of await diffSharedUi( + workspace.workspaceId, + opts.keepDeleted, + )) { if (c.type === "added") { changes.push({ name: "added", path: c.path, content: "" }); } else if (c.type === "deleted") { @@ -5275,7 +5324,9 @@ export async function push( : {}), })), total: changes.length, - ...(changes.length > 0 ? { warning: SYNC_PUSH_DESTRUCTIVE_WARNING } : {}), + ...(changes.length > 0 + ? { warning: syncPushDestructiveWarning(opts.keepDeleted) } + : {}), }; console.log(JSON.stringify(result, null, 2)); return; @@ -5339,7 +5390,9 @@ export async function push( if (opts.dryRun) { log.info(colors.gray(`Dry run complete.`)); - log.warn(colors.yellow(`\n⚠ ${SYNC_PUSH_DESTRUCTIVE_WARNING}`)); + log.warn( + colors.yellow(`\n⚠ ${syncPushDestructiveWarning(opts.keepDeleted)}`), + ); return; } @@ -6313,7 +6366,7 @@ export async function push( } } try { - await pushSharedUi(workspace.workspaceId); + await pushSharedUi(workspace.workspaceId, opts.keepDeleted); } catch (e) { log.warn(`Failed to push shared UI folder: ${e}`); } @@ -6415,7 +6468,10 @@ export async function push( let sharedUiPushed = false; if (!opts.dryRun) { try { - sharedUiPushed = await pushSharedUi(workspace.workspaceId); + sharedUiPushed = await pushSharedUi( + workspace.workspaceId, + opts.keepDeleted, + ); } catch (e) { log.warn(`Failed to push shared UI folder: ${e}`); } @@ -6480,6 +6536,10 @@ const command = new Command() .option("--include-groups", "Include syncing groups") .option("--include-settings", "Include syncing workspace settings") .option("--include-key", "Include workspace encryption key") + .option( + "--keep-deleted", + "Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates.", + ) .option("--skip-branch-validation", "Skip git branch validation and prompts") .option("--json-output", "Output results in JSON format") .option( @@ -6543,6 +6603,10 @@ const command = new Command() "--skip-reencrypt-on-key-change", "When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt.", ) + .option( + "--keep-deleted", + "Do not delete remote items that no longer exist locally. Only adds and updates.", + ) .option("--skip-branch-validation", "Skip git branch validation and prompts") .option("--json-output", "Output results in JSON format") .option( diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index ebd6360875..62027ab2ef 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -7519,6 +7519,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-groups\` - Include syncing groups - \`--include-settings\` - Include syncing workspace settings - \`--include-key\` - Include workspace encryption key + - \`--keep-deleted\` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates. - \`--skip-branch-validation\` - Skip git branch validation and prompts - \`--json-output\` - Output results in JSON format - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes @@ -7550,6 +7551,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-settings\` - Include syncing workspace settings - \`--include-key\` - Include workspace encryption key - \`--skip-reencrypt-on-key-change\` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. + - \`--keep-deleted\` - Do not delete remote items that no longer exist locally. Only adds and updates. - \`--skip-branch-validation\` - Skip git branch validation and prompts - \`--json-output\` - Output results in JSON format - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) diff --git a/cli/test/shared_ui_diff_unit.test.ts b/cli/test/shared_ui_diff_unit.test.ts index f2fd69210e..052fa58a8e 100644 --- a/cli/test/shared_ui_diff_unit.test.ts +++ b/cli/test/shared_ui_diff_unit.test.ts @@ -11,12 +11,25 @@ import * as os from "node:os"; import * as path from "node:path"; let remoteFiles: Record = {}; +let remoteUnreadable = false; +let pushedFiles: Record | undefined; mock.module("../gen/services.gen.ts", () => ({ - getSharedUi: async (_args: { workspace: string }) => ({ files: remoteFiles }), + getSharedUi: async (_args: { workspace: string }) => { + if (remoteUnreadable) throw new Error("shared UI store unreadable"); + return { files: remoteFiles }; + }, + updateSharedUi: async (args: { + workspace: string; + requestBody: { files: Record }; + }) => { + pushedFiles = args.requestBody.files; + }, })); -const { diffSharedUi } = await import("../src/commands/shared_ui.ts"); +const { diffSharedUi, pushSharedUi } = await import( + "../src/commands/shared_ui.ts" +); describe("diffSharedUi", () => { const ws = "test-workspace"; @@ -25,6 +38,8 @@ describe("diffSharedUi", () => { beforeEach(() => { remoteFiles = {}; + remoteUnreadable = false; + pushedFiles = undefined; prevCwd = process.cwd(); tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-shared-ui-")); process.chdir(tmpDir); @@ -89,4 +104,46 @@ describe("diffSharedUi", () => { const changes = await diffSharedUi(ws); expect(changes).toEqual([]); }); + + test("emits nothing under keepDeleted when the remote store is unreadable", async () => { + // pushSharedUi skips the push rather than clearing a store it can't read, + // so the preview must show that same nothing. + remoteUnreadable = true; + writeUi("theme.json", "{}"); + expect(await diffSharedUi(ws, true)).toEqual([]); + }); +}); + +describe("pushSharedUi with keepDeleted", () => { + const ws = "test-workspace"; + let tmpDir: string; + let prevCwd: string; + + beforeEach(() => { + remoteFiles = {}; + remoteUnreadable = false; + pushedFiles = undefined; + prevCwd = process.cwd(); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-shared-ui-push-")); + process.chdir(tmpDir); + }); + + afterEach(() => { + process.chdir(prevCwd); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + test("carries remote-only files, including ui/__proto__, into the pushed map", async () => { + // JSON.parse, not a literal: `{__proto__: …}` sets the prototype instead of + // creating the own property the API response really has. + remoteFiles = JSON.parse('{"__proto__":"keep me","extra.json":"1"}'); + fs.mkdirSync(path.join(tmpDir, "ui"), { recursive: true }); + fs.writeFileSync(path.join(tmpDir, "ui", "theme.json"), "{}", "utf-8"); + + expect(await pushSharedUi(ws, true)).toBe(true); + // The store is written whole, so anything missing here is deleted. + expect(pushedFiles!["extra.json"]).toEqual("1"); + expect(Object.getOwnPropertyDescriptor(pushedFiles!, "__proto__")?.value) + .toEqual("keep me"); + }); }); diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts index 343ed49ef7..24c4a282a3 100644 --- a/cli/test/sync_pull_push.test.ts +++ b/cli/test/sync_pull_push.test.ts @@ -2774,3 +2774,102 @@ kind: script }); }); }); + +describe("keep deleted", () => { + test("Integration: --keep-deleted keeps items absent from the other side", async () => { + await withTestBackend(async (backend, tempDir) => { + const uniqueId = Date.now(); + const scriptPath = `f/test/keep_deleted_${uniqueId}`; + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + content: 'export async function main() { return "keep me"; }', + language: "bun", + summary: "Kept by --keep-deleted", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + } + ); + expect(resp.status).toBeLessThan(300); + await resp.text(); + + await writeWmillYaml(tempDir); + expect( + (await backend.runCLICommand(["sync", "pull", "--yes"], tempDir)).code + ).toEqual(0); + + const contentFile = `${scriptPath}.ts`; + const metadataFile = `${scriptPath}.script.yaml`; + expect(await listFilesRecursive(tempDir)).toContain(contentFile); + + // Push direction: the remote script survives losing its local files. + await rm(join(tempDir, contentFile)); + await rm(join(tempDir, metadataFile)); + expect( + ( + await backend.runCLICommand( + ["sync", "push", "--yes", "--keep-deleted"], + tempDir + ) + ).code + ).toEqual(0); + // A push deletion archives the script rather than removing the row, so + // `archived` — not the status code — is what says it survived. + const remote = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/get/p/${scriptPath}` + ); + expect(remote.status).toEqual(200); + expect((await remote.json()).archived).not.toEqual(true); + + // Pull direction: a file with no remote counterpart survives the pull. + const localOnly = `f/test/local_only_${uniqueId}.ts`; + await writeFile( + join(tempDir, localOnly), + 'export async function main() { return "local only"; }', + "utf-8" + ); + expect( + ( + await backend.runCLICommand( + ["sync", "pull", "--yes", "--keep-deleted"], + tempDir + ) + ).code + ).toEqual(0); + const afterPull = await listFilesRecursive(tempDir); + expect(afterPull).toContain(localOnly); + // Adds still apply: the script deleted above is written back. + expect(afterPull).toContain(contentFile); + + // An empty changeset falls past the dry-run return, on to the shared-UI + // step — which writes to disk, so a dry run must skip it. + // Including the metadata and lock the pull's auto-fill generated for it. + for (const ext of [".ts", ".script.yaml", ".script.lock"]) { + await rm(join(tempDir, `f/test/local_only_${uniqueId}${ext}`), { + force: true, + }); + } + await mkdir(join(tempDir, "ui"), { recursive: true }); + await writeFile(join(tempDir, "ui", "custom.css"), "body{}", "utf-8"); + const dryRun = await backend.runCLICommand( + ["sync", "pull", "--dry-run"], + tempDir + ); + expect(dryRun.code).toEqual(0); + // Guards against a vacuous pass: a non-empty changeset would return at + // the dry-run check above and never reach the shared-UI step. + expect(dryRun.stdout + dryRun.stderr).toContain("0 changes to apply"); + expect(await listFilesRecursive(tempDir)).toContain("ui/custom.css"); + }); + }); +}); diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index 8029ae48c7..e51c8371dc 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -605,6 +605,7 @@ sync local with a remote workspaces or the opposite (push or pull) - `--include-groups` - Include syncing groups - `--include-settings` - Include syncing workspace settings - `--include-key` - Include workspace encryption key + - `--keep-deleted` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates. - `--skip-branch-validation` - Skip git branch validation and prompts - `--json-output` - Output results in JSON format - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes @@ -636,6 +637,7 @@ sync local with a remote workspaces or the opposite (push or pull) - `--include-settings` - Include syncing workspace settings - `--include-key` - Include workspace encryption key - `--skip-reencrypt-on-key-change` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. + - `--keep-deleted` - Do not delete remote items that no longer exist locally. Only adds and updates. - `--skip-branch-validation` - Skip git branch validation and prompts - `--json-output` - Output results in JSON format - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 2c3b585c20..918b88cb21 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -3671,6 +3671,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-groups\` - Include syncing groups - \`--include-settings\` - Include syncing workspace settings - \`--include-key\` - Include workspace encryption key + - \`--keep-deleted\` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates. - \`--skip-branch-validation\` - Skip git branch validation and prompts - \`--json-output\` - Output results in JSON format - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes @@ -3702,6 +3703,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-settings\` - Include syncing workspace settings - \`--include-key\` - Include workspace encryption key - \`--skip-reencrypt-on-key-change\` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. + - \`--keep-deleted\` - Do not delete remote items that no longer exist locally. Only adds and updates. - \`--skip-branch-validation\` - Skip git branch validation and prompts - \`--json-output\` - Output results in JSON format - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index ebd0d7f4da..1f3c54ea26 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -610,6 +610,7 @@ sync local with a remote workspaces or the opposite (push or pull) - `--include-groups` - Include syncing groups - `--include-settings` - Include syncing workspace settings - `--include-key` - Include workspace encryption key + - `--keep-deleted` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates. - `--skip-branch-validation` - Skip git branch validation and prompts - `--json-output` - Output results in JSON format - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes @@ -641,6 +642,7 @@ sync local with a remote workspaces or the opposite (push or pull) - `--include-settings` - Include syncing workspace settings - `--include-key` - Include workspace encryption key - `--skip-reencrypt-on-key-change` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. + - `--keep-deleted` - Do not delete remote items that no longer exist locally. Only adds and updates. - `--skip-branch-validation` - Skip git branch validation and prompts - `--json-output` - Output results in JSON format - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) From b57e231c2bf5e5fe007f0aa7b958a51e32b47141 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Mon, 31 Aug 2026 16:13:32 +0200 Subject: [PATCH 34/40] fix: keep raw-app editor selection consistent across sidebar and tabs (#10885) * fix: route raw-app editor selection through one switch function Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H * fix: stop announcing folders as selected from the file tree Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H * fix: carry the selection through a folder rename Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H * fix: keep the generated wmill.ts tab out of stale-tab cleanup Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H * refactor: test document existence through one predicate Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H * refactor: route the history replay through the same predicate Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H * fix: clear the selection in the same tick a runnable is deleted Deleting the selected runnable dropped it from `runnables` and left the editor to notice via the stale-tab effect, one frame later. In that window the pane rendered "No runnable at id ". The sidebar list now reports the delete instead of mutating `runnables` itself; the editor deletes and closes the tab together, so the selection moves through `select` synchronously. The stale-tab effect stays as the backstop for deletes that come from elsewhere. Also retitle the two sidebar create buttons and rename the FileExplorer exports behind them: both have always anchored on the selected file's parent folder, never the root. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H * refactor: require the runnable delete callback Optional, the row's Delete button renders and does nothing. There is one caller and it always supplies it, so the compiler can hold that. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H --------- Co-authored-by: Claude Opus 5 (1M context) --- .../src/lib/components/FileExplorer.svelte | 73 +++---- .../components/raw_apps/FileTreeNode.svelte | 34 ++- .../components/raw_apps/RawAppEditor.svelte | 195 +++++++++++------- .../RawAppInlineScriptPanelList.svelte | 20 +- .../components/raw_apps/RawAppSidebar.svelte | 41 ++-- frontend/src/lib/components/raw_apps/utils.ts | 3 + 6 files changed, 204 insertions(+), 162 deletions(-) diff --git a/frontend/src/lib/components/FileExplorer.svelte b/frontend/src/lib/components/FileExplorer.svelte index a92972f3e9..e031c21337 100644 --- a/frontend/src/lib/components/FileExplorer.svelte +++ b/frontend/src/lib/components/FileExplorer.svelte @@ -8,9 +8,11 @@ interface Props { /** File path → content map. Keys use / prefix (e.g. /index.html). */ files: Record - /** Currently selected path (/-prefixed). Read-only; changes via onSelectPath callback. */ + /** Currently selected file (/-prefixed). Read-only; changes via onSelectPath callback. */ selectedPath?: string | undefined - /** Called when user clicks a path (file or folder). */ + /** Called when the user clicks a file. Folders aren't selectable — clicking + * one only expands it — so the only non-file paths this reports are the root + * row under `showRoot`, and '' when the last file is deleted. */ onSelectPath?: (path: string) => void /** Extra tree nodes appended after the main tree (e.g. read-only wmill.ts). */ extraNodes?: TreeNode[] @@ -80,6 +82,12 @@ onSelectPath?.(path) } + function parentFolderOfSelection(): string { + if (!selectedPath || selectedPath === '/') return '/' + const pathParts = selectedPath.split('/').filter(Boolean) + return pathParts.length > 1 ? '/' + pathParts.slice(0, -1).join('/') + '/' : '/' + } + function handleAddFile(folderPath: string) { const normalizedFolder = folderPath.endsWith('/') ? folderPath : folderPath + '/' const basePath = normalizedFolder + 'newfile.txt' @@ -88,20 +96,10 @@ pathToEdit = newPath } - export function handleAddRootFile() { - let basePath: string - if (selectedPath && selectedPath !== '/') { - if (selectedPath.endsWith('/')) { - basePath = selectedPath + 'newfile.txt' - } else { - const pathParts = selectedPath.split('/').filter(Boolean) - const parentPath = - pathParts.length > 1 ? '/' + pathParts.slice(0, -1).join('/') + '/' : '/' - basePath = parentPath + 'newfile.txt' - } - } else { - basePath = '/newfile.txt' - } + // New entries land beside the selected file; with nothing selected, at the + // root. To create inside another folder, use that folder row's own menu. + export function handleAddFileBesideSelection() { + const basePath = parentFolderOfSelection() + 'newfile.txt' const newPath = getUniquePath(basePath) pendingNewFilePath = newPath pathToEdit = newPath @@ -115,20 +113,8 @@ pathToEdit = newPath } - export function handleAddRootFolder() { - let basePath: string - if (selectedPath && selectedPath !== '/') { - if (selectedPath.endsWith('/')) { - basePath = selectedPath + 'newfolder/' - } else { - const pathParts = selectedPath.split('/').filter(Boolean) - const parentPath = - pathParts.length > 1 ? '/' + pathParts.slice(0, -1).join('/') + '/' : '/' - basePath = parentPath + 'newfolder/' - } - } else { - basePath = '/newfolder/' - } + export function handleAddFolderBesideSelection() { + const basePath = parentFolderOfSelection() + 'newfolder/' const newPath = getUniquePath(basePath) pendingNewFilePath = newPath pathToEdit = newPath @@ -168,9 +154,7 @@ } // Also rename in emptyFolders emptyFolders = emptyFolders.map((f) => - f === oldPath || f.startsWith(oldPath) - ? newPath + f.substring(oldPath.length) - : f + f === oldPath || f.startsWith(oldPath) ? newPath + f.substring(oldPath.length) : f ) } } else { @@ -187,7 +171,13 @@ files = nfiles pathToEdit = undefined - onSelectPath?.(newPath) + if (!isFolder) { + onSelectPath?.(newPath) + } else if (selectedPath?.startsWith(oldPath)) { + // A folder isn't selectable, but the selected file moved with it — follow + // it to its new path, or the caller keeps editing a key that's now gone. + onSelectPath?.(newPath + selectedPath.slice(oldPath.length)) + } } function handleDelete(path: string) { @@ -208,12 +198,8 @@ files = nfiles if (selectedPath === path || (isFolder && selectedPath?.startsWith(path))) { - const remaining = Object.keys(nfiles) - if (remaining.length > 0) { - onSelectPath?.(remaining[0]) - } else { - onSelectPath?.(showRoot ? '/' : '') - } + const remainingFile = Object.keys(nfiles).find((key) => !key.endsWith('/')) + onSelectPath?.(remainingFile ?? (showRoot ? '/' : '')) } } @@ -223,7 +209,7 @@ Files
    diff --git a/frontend/src/lib/components/raw_apps/FileTreeNode.svelte b/frontend/src/lib/components/raw_apps/FileTreeNode.svelte index 3d428b9aaf..0e02701347 100644 --- a/frontend/src/lib/components/raw_apps/FileTreeNode.svelte +++ b/frontend/src/lib/components/raw_apps/FileTreeNode.svelte @@ -1,5 +1,15 @@ @@ -125,16 +127,8 @@ {runnable} isSelected={selectedRunnable === id} isEditing={editingId === id} - onSelect={() => { - selectedRunnable = id - onSelect?.(id) - }} - onDelete={() => { - delete runnables[id] - if (selectedRunnable === id) { - selectedRunnable = undefined - } - }} + onSelect={() => onSelect?.(id)} + onDelete={() => onDelete(id)} onRename={(newId) => renameRunnable(id, newId)} onRequestEdit={() => (editingId = id)} onCancelEdit={() => (editingId = undefined)} diff --git a/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte b/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte index b8d6e33c87..fbe02e1dc9 100644 --- a/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte @@ -3,6 +3,7 @@ SUBTLE_PANEL_TITLE } from '../apps/editor/settingsPanel/common/PanelSection.svelte' import type { Runnable } from '../apps/inputType' + import { WMILL_TS_PATH } from './utils' import RawAppInlineScriptPanelList from './RawAppInlineScriptPanelList.svelte' import FileExplorer from '../FileExplorer.svelte' import { Plus, File, Folder, Camera } from 'lucide-svelte' @@ -18,10 +19,14 @@ interface Props { runnables: Record + /** Read-only; the editor switches selection through `onSelectRunnable`. */ selectedRunnable: string | undefined files: Record modules?: Modules - onSelectFile?: (path: string) => void + onSelectRunnable?: (key: string) => void + onDeleteRunnable: (key: string) => void + onSelectPath?: (path: string) => void + /** Read-only; the editor switches selection through `onSelectPath`. */ selectedDocument: string | undefined historyManager?: RawAppHistoryManager historySelectedId?: number | undefined @@ -39,11 +44,13 @@ let { runnables, - selectedRunnable = $bindable(), + selectedRunnable, files = $bindable({}), modules, - onSelectFile, - selectedDocument = $bindable(), + onSelectRunnable, + onDeleteRunnable, + onSelectPath, + selectedDocument, historyManager, historySelectedId, onHistorySelect, @@ -79,13 +86,6 @@ } let fileExplorer: FileExplorer | undefined = $state() - - function handleSelectPath(path: string) { - selectedDocument = path - if (!path.endsWith('/')) { - onSelectFile?.(path) - } - }
    -
    + +
    Cases {workingCases.length} @@ -369,13 +372,13 @@ Add a case
    -
    +
    (casesEditing = v)} />
    @@ -397,7 +400,7 @@ unifiedSize="md" variant="accent" loading={saving} - disabled={writing || !path || !!pathError || (nothingToSave && !casesEditing)} + disabled={writing || !path || !!pathError || nothingToSave} onclick={saveDataset} > Save @@ -408,7 +411,8 @@ variant="accent" startIcon={{ icon: Plus }} loading={creating} - disabled={creating || !path || !!pathError} + disabled={creating || !path || !!pathError || noCases} + title={noCasesTitle} onclick={createDataset} > Create dataset diff --git a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte index d4d1714fd0..63fe366628 100644 --- a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte +++ b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte @@ -47,16 +47,6 @@ let hasDraft = $derived(editedConfig !== undefined) let dataset = $state(undefined) let hoveringDataset = $state(false) - /** Set while the dialog stands aside for the dataset drawer, holding the pane's dataset as it - * was on the way out: coming back onto a different one is the detour's answer, and adopted. */ - let steppedAside = $state<{ dataset: string | undefined } | undefined>(undefined) - - /** Hands the screen to the dataset drawer, to be given back when it closes. */ - function stepAside(go: () => void) { - steppedAside = { dataset: defaultDataset } - open = false - go() - } /** The agent's versions, for pinning one. Loaded when the dialog opens rather than held: a * version list goes stale the moment the agent is saved again. */ @@ -76,21 +66,26 @@ } } + /** Whether the dialog was already open on the previous run of the effect below, which reads it + * to tell an open from the pane's dataset moving underneath. */ + let wasOpen = false + $effect(() => { - if (!open) return + const isOpen = open + const pane = defaultDataset untrack(() => { - loadVersions() - const aside = steppedAside - steppedAside = undefined - if (aside) { - // Back from the drawer: a dataset created or edited there moves the pane onto it, and - // anything else leaves the field as it was left. - if (defaultDataset !== aside.dataset) dataset = defaultDataset + if (!isOpen) { + wasOpen = false return } - // Seeded on every open: the dataset last worked in, and the state of the agent there is - // most reason to measure. - dataset = defaultDataset + // Followed while the dialog stands, not only read at open: the dataset drawer opens over + // this dialog rather than in place of it, so creating, renaming or deleting a dataset + // there moves the pane's selection with the dialog still up. Nothing else moves it then. + dataset = pane + if (wasOpen) return + wasOpen = true + loadVersions() + // The state of the agent there is most reason to measure. choice = hasDraft ? 'draft' : 'deployed' }) }) @@ -184,7 +179,7 @@ unifiedSize="sm" variant="default" startIcon={{ icon: Plus }} - onclick={() => stepAside(onNewDataset)} + onclick={onNewDataset} > New dataset @@ -215,7 +210,7 @@ title="Edit this dataset" on:click={() => { close() - stepAside(() => onEditDataset(item.value ?? '')) + onEditDataset(item.value ?? '') }} /> {/snippet} @@ -228,7 +223,7 @@ btnClasses="w-full !h-auto !justify-start !rounded-none flex items-center gap-2 px-3 py-2 text-xs !font-normal text-secondary hover:bg-surface-hover" onClick={() => { close() - stepAside(onNewDataset) + onNewDataset() }} > @@ -247,7 +242,7 @@ startIcon={{ icon: Pencil }} iconOnly title="Edit this dataset" - on:click={() => stepAside(() => onEditDataset(dataset ?? ''))} + on:click={() => onEditDataset(dataset ?? '')} />
    {/if} diff --git a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte index 5d84ef88ce..b544781902 100644 --- a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte +++ b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte @@ -7,17 +7,21 @@ import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import TimeAgo from '$lib/components/TimeAgo.svelte' import { Button } from '$lib/components/common' - import { Bot, Code2, Loader2, Plus } from 'lucide-svelte' + import { Bot, ChevronRight, Code2, Loader2, Plus } from 'lucide-svelte' + import { overlayHostActive, topmostSurface } from '$lib/components/common/overlayHost.svelte' import type { EvalDataset, EvalExperiment, ExperimentScore } from '$lib/gen' import { datasetSummary, experimentName, formatScore, subjectLabel } from './evalUtils' let { experiments, datasets, + caseProgress, loaded, + active = false, deployedHash = undefined, currentVersion = undefined, onOpen, + onHighlight, onEditDataset, onNew }: { @@ -25,17 +29,87 @@ experiments: EvalExperiment[] /** Whether the list has been read: an empty table is a statement about the agent. */ loaded: boolean + /** Whether this list is the page on screen. The keyboard is only answered while it is: the + * run page keeps its own rows, and both would otherwise move on one press. */ + active?: boolean /** The workspace's datasets, for naming the one a run is of by what it is for. */ datasets: EvalDataset[] + /** How many cases each still-running run has finished, keyed by run id. A run the flow has + * not been read for yet is at none of them rather than absent: the count is on the row from + * the moment it appears, so it never arrives late and shifts the column. */ + caseProgress: Record /** What the agent hashes to as deployed, and the version it is on: they resolve a run of * edits that were later saved, so a run is labelled here as the run picker labels it. */ deployedHash?: string currentVersion?: number onOpen: (experiment: EvalExperiment) => void + /** The highlighted run, reported up so the surface can act on it — arrowing into the run + * page opens the run under the highlight rather than whichever was opened last. */ + onHighlight?: (id: string | undefined) => void onEditDataset: (dataset: string) => void onNew: () => void } = $props() + /** The highlighted run, by id. One state for both the pointer and the keyboard, as a melt menu + * does it: hovering a row moves the highlight to it, so the arrows carry on from wherever the + * pointer left off instead of running a second, invisible cursor of their own. It says where + * the highlight is, not what is chosen — a run is not opened until Enter. + * + * By id and not by index: the list is newest-first and the poll prepends to it, so an index + * would quietly come to mean a different run and Enter would open the wrong one. */ + let cursorId = $state(undefined) + let cursor = $derived( + cursorId === undefined ? -1 : experiments.findIndex((e) => e.id === cursorId) + ) + let body: HTMLTableSectionElement | undefined = $state() + + // A window listener answers keys aimed anywhere, so it has to ask two questions the DOM cannot: + // is my host the visible one — session preview tabs stay mounted when hidden — and is my surface + // still the one on top, rather than under a drawer or a dialog opened since. + const hostActive = overlayHostActive() + const onTop = topmostSurface() + const listening = () => hostActive() && onTop() + + $effect(() => { + onHighlight?.(cursorId) + }) + + // A highlight on a run that has since gone, and the highlight itself when the list is not the + // page on screen. + $effect(() => { + if (!active || (cursorId !== undefined && cursor < 0)) cursorId = undefined + }) + + function move(by: number) { + if (experiments.length === 0) return + const from = cursor < 0 ? (by > 0 ? -1 : experiments.length) : cursor + const at = Math.max(0, Math.min(experiments.length - 1, from + by)) + cursorId = experiments[at]?.id + // `nearest`, so arrowing through a long list scrolls by a row rather than jumping the table. + requestAnimationFrame(() => + body?.querySelectorAll('tr')[at]?.scrollIntoView({ block: 'nearest' }) + ) + } + + function onKeydown(event: KeyboardEvent) { + if (!active || !listening() || event.metaKey || event.ctrlKey || event.altKey) return + const el = event.target as HTMLElement | null + if (el?.closest?.('input, textarea, select, [contenteditable="true"], [role="listbox"]')) return + if (event.key === 'ArrowDown') { + event.preventDefault() + move(1) + } else if (event.key === 'ArrowUp') { + event.preventDefault() + move(-1) + } else if (event.key === 'Enter' && experiments[cursor]) { + // Enter belongs to whatever is focused if that thing does something with it. A highlighted + // row is not a reason to swallow the press on `New evaluation` or a row's dataset button. + if (el?.closest?.('button, a[href], [role="button"], summary')) return + event.preventDefault() + onOpen(experiments[cursor]) + } + } + /** The one number a column reports: a pass rate where it has a line to pass, the mean where it * does not. */ function headline(score: ExperimentScore): string | undefined { @@ -44,13 +118,16 @@ } + + - + + @@ -58,12 +135,20 @@ Dataset Cases Scores - When + When + - - {#each experiments as experiment (experiment.id)} - onOpen(experiment)}> + + {#each experiments as experiment, i (experiment.id)} + + onOpen(experiment)} + on:hover={(e) => e.detail && (cursorId = experiment.id)} + >
    @@ -99,7 +184,18 @@ - {experiment.case_count} + {#if experiment.running} + + + + {caseProgress[experiment.id] ?? 0}/{experiment.case_count} + + {:else} + {experiment.case_count} + {/if}
    @@ -117,8 +213,6 @@ {value} {:else if score.failed > 0} failed - {:else if experiment.running} - {:else} — {/if} @@ -126,33 +220,34 @@ {/each} {#if (experiment.scores ?? []).length === 0} - {#if experiment.running} - - - scoring - - {:else} - not scored - {/if} + + not scored {/if}
    - + + + + {/each} {#if experiments.length === 0 && !loaded} - + {:else if experiments.length === 0} - +
    No runs yet diff --git a/frontend/src/lib/components/aiEvals/EvalScorers.svelte b/frontend/src/lib/components/aiEvals/EvalScorers.svelte index 68287b0fbc..d00e52d77e 100644 --- a/frontend/src/lib/components/aiEvals/EvalScorers.svelte +++ b/frontend/src/lib/components/aiEvals/EvalScorers.svelte @@ -196,26 +196,31 @@
    -
    +
    {#if scorers.length === 0} -
    +
    A scorer reads one run and returns a number. Every run of this dataset is measured by all of them, which is what makes two runs comparable.
    {:else} -
    + +
    {#each scorers as scorer (scorer.id)} -
    +
    {#if scorer.kind === 'agent'} - + {:else} - + {/if}
    - + {scorerLabel(scorer)} - {scorer.path} + {scorer.path}
    {#if scorer.pass_if != undefined} diff --git a/frontend/src/lib/components/aiEvals/EvalsPane.svelte b/frontend/src/lib/components/aiEvals/EvalsPane.svelte index d9e5b16fe4..55792ba8e2 100644 --- a/frontend/src/lib/components/aiEvals/EvalsPane.svelte +++ b/frontend/src/lib/components/aiEvals/EvalsPane.svelte @@ -9,9 +9,11 @@ import Label from '$lib/components/Label.svelte' import Popover from '$lib/components/Popover.svelte' import { Splitpanes, Pane } from 'svelte-splitpanes' + import AnimatedPane from '$lib/components/splitPanes/AnimatedPane.svelte' import { type AgentDraft, AiEvalsService, + JobService, type EvalCase, type EvalDataset, type EvalExperiment, @@ -35,9 +37,11 @@ Code2, ExternalLink } from 'lucide-svelte' + import PagedContent from '$lib/components/common/modal/PagedContent.svelte' import EvalDatasetDrawer from './EvalDatasetDrawer.svelte' import EvalRunsList from './EvalRunsList.svelte' import EvalRunDialog from './EvalRunDialog.svelte' + import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import GfmMarkdown from '$lib/components/GfmMarkdown.svelte' import { caseLabel, @@ -52,6 +56,10 @@ /** A dataset is capped at this many cases, so one page holds the whole set. */ const CASE_PAGE_SIZE = 1000 + /** The id the run's flow gives the loop over its cases (`CASES_NODE_ID` in `ai_evals/run.rs`). + * Looked up by id rather than by position: the flow has a step after the loop too. */ + const CASES_MODULE_ID = 'cases' + let { agentPath, opWorkspace = undefined, @@ -91,12 +99,56 @@ /** What the agent hashes to as deployed: a run of edits carrying it ran what was then saved. */ let deployedHash = $state(undefined) let running = $state(false) - let scorers = $derived(dataset?.scorers ?? []) + /** The run on screen belongs to a dataset still being read. Until it arrives the rows and the + * scorer columns would both be built from the *previous* dataset, so both are held back. */ + let datasetLoading = $state(false) + let scorers = $derived(datasetLoading ? [] : (dataset?.scorers ?? [])) let selectedCaseId = $state(undefined) let datasetDrawer: EvalDatasetDrawer | undefined = $state() let runDialogOpen = $state(false) - let resumeRunDialog = $state(false) + /** The run the list has highlighted, so arrowing into the run page opens that one. Without it + * the arrow could only fall back to whichever run was opened last, which on a dialog just + * opened is none at all. */ + let highlightedRunId = $state(undefined) + + /** How many cases each still-running run has finished, keyed by run id. Read from the flow + * executing the run: the list carries the case total, and counting the finished ones there + * would be a per-case query for every run listed. The flow already records it — one slot per + * case in `flow_jobs_success`, null until that case's iteration is over. */ + let caseProgress = $state>({}) + + async function readCaseProgress() { + const workspace = ws + const live = experiments.filter((e) => e.running) + if (!workspace || live.length === 0) { + if (Object.keys(caseProgress).length > 0) caseProgress = {} + return + } + const read = await Promise.all( + live.map(async (e) => { + try { + const update = await JobService.getJobUpdates({ + workspace, + id: e.run_job_id, + running: true, + noLogs: true + }) + const cases = update.flow_status?.modules?.find((m) => m.id === CASES_MODULE_ID) + if (!cases) return undefined + return [ + e.id, + (cases.flow_jobs_success ?? []).filter((s) => s != undefined).length + ] as const + } catch { + // Left out of the map, so the row reads `0/total` until a later poll answers. A + // flow that cannot be read is already the list's problem to report, not this one's. + return undefined + } + }) + ) + caseProgress = Object.fromEntries(read.filter((e) => e !== undefined)) + } let experiment = $derived(experiments.find((e) => e.id === experimentId)) @@ -120,6 +172,7 @@ runsLoadError = false try { experiments = await listSubjectExperiments() + await readCaseProgress() } catch (e) { runsLoadError = true sendUserToast(`Failed to load the runs: ${e}`, true) @@ -183,6 +236,8 @@ // Switching datasets leaves the previous request in flight; only the newest may write, or a // slow response for the dataset you just left replaces the one you are looking at. let loadGeneration = 0 + /** Which run the pane is opening; only the newest may clear `datasetLoading`. */ + let openGeneration = 0 async function loadDataset(path: string | undefined): Promise { const generation = ++loadGeneration @@ -345,6 +400,7 @@ await loadResults() } else { experiments = await listSubjectExperiments() + await readCaseProgress() } } finally { refreshing = false @@ -354,27 +410,46 @@ /** Opens a run, bringing its dataset with it and offering the run before it as the baseline. * Reading the cells is left to the effect on the selection, so every way in opens one alike. */ async function openRun(id: string) { - // Against the dataset that is loaded, not the one that is selected: skipping on the selection - // alone would leave a run open over a dataset whose cases and scorers were never read. const target = experiments.find((e) => e.id === id) - if (target && target.dataset !== dataset?.path) { - await useDataset(target.dataset) + // Only when the run itself changes: re-showing the one already open — arrowing back into it + // from the list — must keep whatever comparison the user picked. + if (id !== experimentId) { + const index = experiments.findIndex((e) => e.id === id) + // The run before it *of the same dataset*: the list spans datasets, and a run of another + // set of cases is not a baseline for this one. + baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id } - const index = experiments.findIndex((e) => e.id === id) - // The run before it *of the same dataset*: the list spans datasets, and a run of another set - // of cases is not a baseline for this one. - baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id experimentId = id - viewingRun = true selectedCaseId = undefined + // Opened first, read second: the dataset is a request, and waiting on it here is a click + // that does nothing at all until the network answers. The page carries the wait instead. + // + // Against what is *selected* as well as what is loaded. `selectedDataset` moves the moment a + // read starts, so a load still in flight for another dataset shows up here: without that + // test, opening a run of the dataset already committed would skip `useDataset` entirely and + // leave the in-flight one free to commit its cases under this run. + const needsDataset = + !!target && (target.dataset !== dataset?.path || target.dataset !== selectedDataset) + datasetLoading = needsDataset + viewingRun = true + if (needsDataset) { + // Numbered like `loadDataset`'s own read, and for the same reason: opening a second run + // while the first is still loading leaves two `finally`s racing, and the loser clearing + // the flag would uncover the table with neither dataset in hand. + const generation = ++openGeneration + try { + await useDataset(target!.dataset) + } finally { + if (generation === openGeneration) datasetLoading = false + } + } } async function runAll(runSubject: EvalSubject, path: string): Promise { if (!ws || !path) return false running = true - let id: string try { - id = await AiEvalsService.runExperiment({ + await AiEvalsService.runExperiment({ workspace: ws, requestBody: { dataset: path, subject: runSubject } }) @@ -386,9 +461,10 @@ // From here the run exists and is billing: what can still fail is reading it back, and // saying "failed to run" to that invites a second, duplicate run. try { + // Onto the list rather than into the run: a run that has just started has no answers and + // no scores, and the list already fills its row in as they land. Reading it is a click. if (path !== dataset?.path) await useDataset(path) await loadRuns() - await openRun(id) } catch (e) { sendUserToast( `The run started but could not be read back: ${e}. Reload the runs list to see it.`, @@ -407,8 +483,6 @@ /** The dataset is gone and every run of it with it: back to the list, on no dataset. */ async function datasetDeleted(path: string) { - // A run dialog waiting behind the drawer has nothing to come back to. - resumeRunDialog = false if (selectedDataset === path) { viewingRun = false selectedCaseId = undefined @@ -442,6 +516,13 @@ } let selectedRow = $derived(displayRows.find((row) => row.case_id === selectedCaseId)) + /** The case the side panel is showing. Held rather than read straight off the selection: the + * pane animates shut over a few hundred milliseconds, and the selection is gone on the first + * of them, which would empty the panel before it had finished closing. */ + let openRow = $state(undefined) + $effect(() => { + if (selectedRow) openRow = selectedRow + }) $effect(() => { if (!ws) return @@ -570,34 +651,52 @@
    + {#if loaded && loadError} +
    + Could not load evals + + The datasets or runs could not be read. Check your access to this agent and reload. + +
    + {:else} + + { + // Right opens the run under the highlight, falling back to whichever was open before; + // left is the way back, the same as the breadcrumb. + if (key === 'run') { + // Both branches go through `openRun`: it is what brings the run's own dataset back, + // and the fallback run may be of a dataset the list has since moved off. + const id = highlightedRunId ?? experimentId + if (id) openRun(id) + } else if (key === 'list') { + viewingRun = false + selectedCaseId = undefined + } + }} + pages={[ + { key: 'list', content: listPage }, + { key: 'run', content: runPage } + ]} + /> + {/if} +
    + +{#snippet listPage()} + +

    + Each run answers a dataset of cases with this agent and scores the answers, so runs can be + compared. +

    - {#if viewingRun} - - {/if}
    - {#if viewingRun && experiment?.run_job_id} - - Open the job - - - {/if} - {#if !viewingRun && loaded && datasets.length > 0} + {#if loaded && datasets.length > 0} -
    - {:else if !viewingRun || !loaded} - openRun(e.id)} - onEditDataset={async (path) => { - if (await useDataset(path)) datasetDrawer?.openDrawer('edit') - }} - onNew={() => (runDialogOpen = true)} - /> - {:else} - - - - - {#each scorers as scorer (scorer.id)} - - {/each} - - - - Case - Answer - {#each scorers as scorer, index (scorer.id)} - {@const mean = means.find((m) => m.scorer_id === scorer.id)} - {@const headline = columnHeadline(scorer, mean)} - - -
    - - {#if scorer.kind === 'agent'} - - {:else} - - {/if} - {scorerLabel(scorer)} - - - {#if headline} - - {headline.value} + + + + + {#each scorers as scorer (scorer.id)} + + {/each} + + + + Case + Answer + {#each scorers as scorer, index (scorer.id)} + {@const mean = means.find((m) => m.scorer_id === scorer.id)} + {@const headline = columnHeadline(scorer, mean)} + + +
    + + {#if scorer.kind === 'agent'} + + {:else} + + {/if} + {scorerLabel(scorer)} + + + {#if headline} + + {headline.value} + + {#if headline.delta && headline.direction !== 0} + 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} + > + {headline.delta} - {#if headline.delta && headline.direction !== 0} - 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} - > - {headline.delta} - - {/if} {/if} - -
    -     - {/each} + {/if} +     +
    +     + {/each} + + + + {#if datasetLoading} + + + + + - - + {:else} {#each displayRows as row (row.case_id)} {@const status = statusOf(row.status)} {/each} - -     - {/if} + {/if} + +
    - {#if selectedRow} - {@const openRow = selectedRow} - + + + {#if openRow}
    -
    - + +
    + {openRow.input?.user_message ?? caseLabel(openRow)} -
    - {#if openRow.job_id} - - Open the case job - - - {/if} -
    -
    + +
    {#if openRow.expected != undefined && openRow.expected !== ''} - +
    +
    + Expected +
    +
    + + {typeof openRow.expected === 'string' + ? openRow.expected + : JSON.stringify(openRow.expected, null, 2)} + +
    +
    {/if} {#if scorers.length > 0 && openRow.scores.length > 0} -
    -
    +{/snippet} { - if (await useDataset(path)) { - resumeRunDialog = true - datasetDrawer?.openDrawer('edit') - } - }} - onNewDataset={() => { - resumeRunDialog = true - datasetDrawer?.openDrawer('new') + if (await useDataset(path)) datasetDrawer?.openDrawer('edit') }} + onNewDataset={() => datasetDrawer?.openDrawer('new')} /> { - if (!resumeRunDialog) return - resumeRunDialog = false - // On the dataset the drawer was just in: the dialog opens on the pane's own, which - // creating or editing one has already moved to it. - runDialogOpen = true - }} /> + + diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css deleted file mode 100644 index 1ac9f8b31c..0000000000 --- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css +++ /dev/null @@ -1,26 +0,0 @@ -/* MultilineCellEditor: a popup positioned over the cell, so it has to paint the cell's own frame - rather than inherit it. */ -.ag-theme-alpine .wm-multiline-cell-editor, -.ag-theme-alpine-dark .wm-multiline-cell-editor { - background-color: var(--ag-background-color); -} -.ag-theme-alpine .wm-multiline-cell-editor textarea, -.ag-theme-alpine-dark .wm-multiline-cell-editor textarea { - display: block; - box-sizing: border-box; - /* Horizontal only: the vertical padding is set by the editor, which knows the height of the row - it is replacing. `line-height` here is what it computes against. */ - padding: 0 calc(var(--ag-cell-horizontal-padding) - 1px); - border: 1px solid var(--ag-input-focus-border-color); - border-radius: 3px; - outline: none; - resize: none; - /* Past this it scrolls rather than growing. */ - max-height: 40vh; - overflow-y: auto; - background-color: var(--ag-background-color); - color: var(--ag-foreground-color); - font: inherit; - line-height: 20px; - white-space: pre-wrap; -} diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts deleted file mode 100644 index 00975955c9..0000000000 --- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts +++ /dev/null @@ -1,108 +0,0 @@ -import type { ColDef, ICellEditorComp, ICellEditorParams } from 'ag-grid-community' -// Beside the editor rather than in the AgGrid theme: that file is the vendored theme, and a rule -// added to it is one the next copy of it drops. -import './multilineCellEditor.css' - -/** Kept in step with the `line-height` the stylesheet gives the textarea. */ -const LINE_HEIGHT = 20 - -/** - * A text cell editor that starts the height of the cell and grows as lines are added, for columns - * holding prose rather than a value. Enter commits, Shift+Enter adds a line, Escape cancels. - * - * Rendered as a popup positioned over the cell: an in-cell editor is clipped to the row height, so - * growing is only visible if the editor is allowed to paint outside it. - */ -export class MultilineCellEditor implements ICellEditorComp { - private eGui!: HTMLDivElement - private textarea!: HTMLTextAreaElement - private params!: ICellEditorParams - private wasEmpty = false - - init(params: ICellEditorParams) { - this.params = params - this.eGui = document.createElement('div') - this.eGui.className = 'wm-multiline-cell-editor' - - this.wasEmpty = params.value == undefined - - this.textarea = document.createElement('textarea') - this.textarea.rows = 1 - // A keystroke that opened the edit replaces the value, as it does in every other cell; F2 - // and double-click keep it to be edited. - this.textarea.value = params.eventKey?.length === 1 ? params.eventKey : (params.value ?? '') - this.textarea.style.width = `${params.column.getActualWidth() - 2}px` - // Padded so one line fills the cell it replaces and a second costs a line rather than a row. - // From the row rather than from `--ag-row-height`, which is the theme's figure and not - // necessarily this grid's. - const rowHeight = params.node.rowHeight ?? 28 - const padding = Math.max(0, (rowHeight - LINE_HEIGHT - 2) / 2) - this.textarea.style.paddingTop = `${padding}px` - this.textarea.style.paddingBottom = `${padding}px` - - this.textarea.addEventListener('input', () => this.resize()) - this.textarea.addEventListener('keydown', (e) => { - if (e.key === 'Escape') { - // Kept from whatever is around the grid: a grid in a drawer or a dialog is under a - // surface that closes on Escape, and leaving an edit is not asking to leave that. - e.preventDefault() - e.stopPropagation() - this.params.api.stopEditing(true) - return - } - if (e.key !== 'Enter' || e.isComposing) return - // Both branches keep the key from the grid, which ends the edit on Enter whether or not - // Shift is held: Shift+Enter falls through to the textarea's own newline, and plain Enter - // ends the edit here instead. - e.stopPropagation() - if (!e.shiftKey) { - e.preventDefault() - this.params.stopEditing() - } - }) - this.eGui.appendChild(this.textarea) - } - - private resize() { - this.textarea.style.height = 'auto' - this.textarea.style.height = `${this.textarea.scrollHeight}px` - } - - getGui() { - return this.eGui - } - - afterGuiAttached() { - this.resize() - this.textarea.focus() - // At the end rather than selected: a selection is a keystroke away from erasing the cell. - const end = this.textarea.value.length - this.textarea.setSelectionRange(end, end) - } - - getValue() { - // Nothing typed into a cell that held nothing is not an edit: returning '' here would write - // an empty string over a null, which the grid would see as a change and commit. - if (this.wasEmpty && this.textarea.value === '') return this.params.value - return this.textarea.value - } - - isPopup() { - return true - } - - getPopupPosition(): 'over' | 'under' { - return 'over' - } -} - -/** - * What a column of prose needs, ready to spread into a colDef. `suppressKeyboardEvent` as well as - * the editor: the grid ends an edit on Enter from a handler a popup editor's DOM does not sit - * under, so the editor cannot keep Shift+Enter for itself on its own. - */ -export const multilineCellColDef: Pick = { - cellEditor: MultilineCellEditor, - suppressKeyboardEvent: (p) => - p.editing && (p.event as KeyboardEvent).key === 'Enter' && (p.event as KeyboardEvent).shiftKey -} diff --git a/frontend/src/lib/components/common/EditableTextarea.svelte b/frontend/src/lib/components/common/EditableTextarea.svelte new file mode 100644 index 0000000000..26a6574027 --- /dev/null +++ b/frontend/src/lib/components/common/EditableTextarea.svelte @@ -0,0 +1,173 @@ + + + +{#if editing} + + +{:else} + + +{/if} diff --git a/frontend/src/lib/components/common/drawer/Drawer.svelte b/frontend/src/lib/components/common/drawer/Drawer.svelte index d375bf506e..6bb5358940 100644 --- a/frontend/src/lib/components/common/drawer/Drawer.svelte +++ b/frontend/src/lib/components/common/drawer/Drawer.svelte @@ -8,6 +8,7 @@ import { onMount, createEventDispatcher, setContext, untrack } from 'svelte' import { BROWSER } from 'esm-env' import Disposable from './Disposable.svelte' + import { setTopmostSurface } from '$lib/components/common/overlayHost.svelte' import ConditionalPortal from './ConditionalPortal.svelte' import { chatState } from '$lib/components/copilot/chat/sharedChatState.svelte' import { useReducedMotion } from '$lib/svelte5Utils.svelte' @@ -51,6 +52,11 @@ let disposable: Disposable | undefined = $state(undefined) + // A drawer stacks like a dialog does, so content inside it gets the same answer about whether + // its keys are meant for it. Without this, a drawer opened over a dialog would inherit the + // dialog's answer — false, because the drawer itself is now on top — and go deaf. + setTopmostSurface(() => disposable?.isTopmost() ?? true) + let reducedMotion = useReducedMotion() let duration = $derived(reducedMotion.val ? 0 : _duration) let durationMs = $derived(duration * 1000) diff --git a/frontend/src/lib/components/common/emptyState/EmptyState.svelte b/frontend/src/lib/components/common/emptyState/EmptyState.svelte index 8715065ad0..49c3cd2b77 100644 --- a/frontend/src/lib/components/common/emptyState/EmptyState.svelte +++ b/frontend/src/lib/components/common/emptyState/EmptyState.svelte @@ -10,6 +10,16 @@ label: string icon?: any onClick: () => void + /** + * `default` unless the surface has no other live call to action. Accent is for the case + * where this button is the only thing to press — a form whose submit is disabled until + * this is done, say — so it is not competing with one. + */ + variant?: 'default' | 'accent' + /** Same write lock the surface's other controls take. An empty state is still a live + * control: without this it stays clickable while a request that has already read the + * empty list is in flight, and whatever it adds is discarded when that request lands. */ + disabled?: boolean aiId?: string aiDescription?: string } @@ -32,11 +42,13 @@ {/if}
    {#if action} - +
    diff --git a/frontend/src/lib/components/common/modal/PagedContent.svelte b/frontend/src/lib/components/common/modal/PagedContent.svelte new file mode 100644 index 0000000000..430a53e66f --- /dev/null +++ b/frontend/src/lib/components/common/modal/PagedContent.svelte @@ -0,0 +1,202 @@ + + + + + + + +
    + {#each pages as page, i (page.key)} + {#if visited.includes(page.key) || warmed} + + +
    + {@render page.content()} +
    + {/if} + {/each} +
    + + diff --git a/frontend/src/lib/components/common/overlayHost.svelte.ts b/frontend/src/lib/components/common/overlayHost.svelte.ts index 0e76709cbf..7c4374115b 100644 --- a/frontend/src/lib/components/common/overlayHost.svelte.ts +++ b/frontend/src/lib/components/common/overlayHost.svelte.ts @@ -72,3 +72,23 @@ export function overlayHostActive(): () => boolean { const host = getOverlayHost() return () => host?.active() ?? true } + +const TOPMOST_SURFACE_KEY = 'topmostSurface' + +/** + * Declare whether the surface enclosing this subtree is the one on top. Set by whatever owns the + * stacking — a dialog, a drawer — so content inside it can tell a key meant for itself from one + * meant for something opened over it. + */ +export function setTopmostSurface(isTopmost: () => boolean) { + setContext(TOPMOST_SURFACE_KEY, isTopmost) +} + +/** + * Whether the enclosing surface is on top. True when nothing declared otherwise, so content that + * is not inside such a surface is not silently made deaf. + */ +export function topmostSurface(): () => boolean { + const isTopmost = getContext<(() => boolean) | undefined>(TOPMOST_SURFACE_KEY) + return () => isTopmost?.() ?? true +} diff --git a/frontend/src/lib/components/text_input/TextInput.svelte b/frontend/src/lib/components/text_input/TextInput.svelte index c57f57120a..03fcedbd8e 100644 --- a/frontend/src/lib/components/text_input/TextInput.svelte +++ b/frontend/src/lib/components/text_input/TextInput.svelte @@ -81,6 +81,11 @@ size?: ButtonType.UnifiedSize unifiedHeight?: boolean underlyingInputEl?: UnderlyingInputElT + /** + * Passed to the `autosize` action on the `textarea` variant. Chiefly `minHeight: 0`, for a + * field that hugs one line instead of reserving the action's 30px floor. + */ + autosizeParams?: import('$lib/autosize').AutosizeParams } export function focus() { @@ -108,7 +113,8 @@ error, size = 'md', unifiedHeight = true, - underlyingInputEl: _underlyingInputEl + underlyingInputEl: _underlyingInputEl, + autosizeParams }: Props = $props() let underlyingInputEl = $derived(_underlyingInputEl ?? ('input' as const)) @@ -152,7 +158,7 @@ onpointerdown={(e) => e.stopImmediatePropagation()} bind:this={inputEl} bind:value - use:autosize + use:autosize={autosizeParams} > {:else if underlyingInputEl === 'input'} Date: Mon, 31 Aug 2026 20:09:53 +0200 Subject: [PATCH 39/40] feat: free AI tokens + home search/filter revamp (#10020) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add free Claude Opus tier with per-user token limit Co-Authored-By: Claude Opus 4.8 (1M context) * nit move alert * Home AI Chat * wire home ai chat * auto send prompt * refactor: remove keyboard arrow-navigation from home list Co-Authored-By: Claude Opus 4.8 (1M context) * feat: replace home search bar with unified FilterSearchbar Co-Authored-By: Claude Opus 4.8 (1M context) * feat: replace home quick tags with FilterSearchbar presets Co-Authored-By: Claude Opus 4.8 (1M context) * feat: add content filter to home FilterSearchbar with EE-gated content view - Clear the kind filter by deleting the key (was showing a 'kind: null' tag on All) - Remove the standalone Content button - Add a 'content' filter; when set, render the Ctrl-K content-search view (ContentSearchInner) which shows text-match snippets and its own EE warning Co-Authored-By: Claude Opus 4.8 (1M context) * feat: disable home AI chat and prompt to configure AI when no model Co-Authored-By: Claude Opus 4.8 (1M context) * track cost instead of tokens * nit * fix: load copilot config on home so AI chat isn't wrongly gated Co-Authored-By: Claude Opus 4.8 (1M context) * Home page update * nits * example prompts * nit * feat: switch free AI tier to DeepSeek with daily cost budgets Co-Authored-By: Claude Opus 4.8 (1M context) * nit * Move bottom buttons to HomeAIChat * [ee] feat: surface free AI tier state and make its metering abort-proof Makes the free Windmill AI tier legible to the user and closes an abuse hole. Backend: - AIConfig gains a response-only free_tier marker (skip_deserializing so a client can't store a forged one via edit_copilot_config). get_copilot_info keeps returning it once the grant is spent, so the client knows AI is off because the grant ran out, not because nothing was configured. - Per-user grant becomes one-time (migration drops the day key from ai_free_token_usage); the daily table stays as the instance kill-switch. - Reserve-then-reconcile metering (see EE commit) so a mid-stream disconnect can no longer dodge the usage report and get metered zero. Frontend: - copilotInfo carries freeTier; model settings show a "Free" pill and a usage meter that warns past 80%. - The home chat and the session chat show a dedicated "you've used your free Windmill AI, add your own API key" state instead of the generic "no provider configured" one. - A failed send re-fetches copilot_info so the exhausted state (and its banner) appears live, without a page reload. Bumps ee-repo-ref.txt to the matching EE commit. Co-Authored-By: Claude Opus 4.8 (1M context) * feat: free AI usage meter reusing the context-usage gauge Show free-tier spend with the same gauge as context usage instead of a bespoke block: - Extract the meter+tooltip into a shared UsageMeter; ContextUsageIndicator uses it, and a new FreeTierUsageIndicator renders it from copilotInfo.freeTier. Placed in the session-chat toolbar and next to the home-chat model settings; the old meter block in the model-settings dropdown is removed (the "Free" pill stays). - Hide the context-usage bar while on the free tier so the free meter takes that slot. - Refresh copilotInfo after every free-tier turn (AIChatManager finally) so the meter advances live and the turn that exhausts the grant flips to the exhausted state, instead of both only updating on reload. Gated to active free-tier users, so it costs nothing for configured-key users. Co-Authored-By: Claude Opus 4.8 (1M context) * docs: fix stale free-tier comments after DeepSeek/cost rework Co-Authored-By: Claude Opus 4.8 (1M context) * feat: always show context bar, replace free-tier meter with usage banner Co-Authored-By: Claude Opus 4.8 (1M context) * nit * fix: atomic free-tier budget reservation (ee ref + sqlx) Co-Authored-By: Claude Opus 4.8 (1M context) * fix: keep CLI/MCP and Hub buttons unblurred on AI chat hover Co-Authored-By: Claude Opus 4.8 (1M context) * Add back arrow nav * nit * nit * fix: three review P1s in the home AI chat & search - AIChatManager: refreshFreeTierUsage now bails unless the global copilot state still belongs to the completing manager's workspace, so a warm session finishing after a workspace switch can't reload its (background) workspace over the active one's models/client/copilotWorkspace. - HomeAIChat: block submission until the copilot config is loaded AND enabled (new `canSend`), so a prompt submitted during the unknown-config window isn't handed to a session that never sends it and silently lost. The disabled overlay still gates on config-loaded to avoid a flash. - ItemsList: the content-search reload effect now depends on $workspaceStore so content results follow the active workspace instead of showing the previous one's. Co-Authored-By: Claude Opus 4.8 (1M context) * [ee] fix: harden the three home-AI-chat/search P1s after deeper review Follow-up to the previous P1 commit; sharper review found the earlier guards insufficient: - refreshFreeTierUsage now compares against the most-recently-*requested* workspace (new copilotWorkspaceRequested in aiStore, set synchronously in loadCopilot), not the last-*resolved* one — otherwise a warm session finishing while a newer workspace's load is still in flight could win the monotonic token and restore its stale workspace over the one being loaded. - The content-search view is keyed by workspace ({#key $workspaceStore}) so a switch remounts ContentSearchInner; late in-flight responses from the previous workspace can no longer land in the new one's component. Backend (EE, via ee-repo-ref bump to 03ef0eb): the free-tier reservation now also prices the worst-case input cap (at the cache-miss rate), and enforce_free_tier_body rejects oversized prompts and pins n=1 — so an aborted large-prompt request can no longer dodge the input bill that reconciliation would otherwise charge. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: exclude service accounts from the free AI tier Free-tier eligibility was keyed solely on authed.email. Workspace admins can create and impersonate arbitrary service accounts (synthetic *.sa.wm.dev identities), each of which would receive its own one-time grant — letting one tenant mint many grants and drain the instance-wide daily allowance. Skip the free-tier fallback for *.sa.wm.dev identities. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: activate free AI tier when clearing a workspace provider edit_copilot_config returned AIConfig::default() when the saved workspace config had no providers and no instance config existed; the frontend applies that response immediately, disabling AI even though the free-tier key is available. A later get_copilot_info (on reload) returns the synthetic free-tier config, so clearing a provider behaved inconsistently until reload. Give this response path the same free-tier fallback as get_copilot_info. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: gate the home AI composer behind the global-AI dev flag The "Build with AI" composer starts a session and navigates to /sessions, which lives behind the same wm_dev_global_ai dev gate as the global AI chat. With the gate off (the default), /sessions renders only its gate message, SessionWrapper never mounts, and the queued prompt is silently dropped. Hide the home entry point behind isGlobalAiEnabled() so it isn't exposed before the sessions gate opens. Co-Authored-By: Claude Opus 4.8 (1M context) * [ee] chore: bump ee-repo-ref for deepseek-v4-flash price/model fix Points at the EE commit that pins deepseek-v4-flash and its real prices (pico-precision accounting). Co-Authored-By: Claude Opus 4.8 (1M context) * [ee] fix: provable byte bound for the free-tier input cap (ee-repo-ref) Bumps ee-repo-ref to the EE commit that caps the raw request body byte length directly (token_count <= byte_count is provable), replacing the unsafe body.len()/2 token estimate that high-entropy prompts could beat. Co-Authored-By: Claude Opus 4.8 (1M context) * nit isGlobalAiEnabled * empty commit * fix(frontend): address Codex review on free-tier / home filters - P1: home filters now sync from the URL reactively, so browser Back/Forward updates the chips, kind toggle and results (and clears keys dropped from the URL) instead of leaving them stale until the next filter edit. - Free-tier banner buttons drop deprecated Button props (size/color/border variant) for unifiedSize + a supported variant. - Condense refreshFreeTierUsage comments to a single race-condition constraint beside the guard. Co-Authored-By: Claude Opus 4.8 * fix(frontend): hide empty kind badge on draft-only scripts A draft-only script can carry an empty `kind`, which still isn't 'script' so the row rendered a blue badge whose only content was capitalize('') — an empty pill left of the "Draft only" badge. Guard the badge on a non-empty kind. Co-Authored-By: Claude Opus 4.8 * feat(frontend): animate home tree-view group expand/collapse Wrap each owner group's children in ResizeTransitionWrapper so height changes animate. A slide transition only animates the initial mount, but a freshly-opened owner fetches its rows and passes through a transient empty state before they land — the ResizeObserver animates that second growth too. Nested TreeViews inherit the wrapper's context and skip their own, so one observer per top-level owner animates the whole subtree. Co-Authored-By: Claude Opus 4.8 * feat(frontend): FilterSearchbar boolean auto-set and string-filter presets - A default-false boolean filter has only one useful value, so selecting it sets true immediately instead of opening a true/false picker. A default-true boolean (e.g. "Include library scripts") still shows the picker, where false is the meaningful choice — expressed via a new optional `default` on the schema. - A plain string filter now surfaces any presets targeting it (`:`) as suggestions once selected, integrated into menuItems so keyboard nav works — previously selecting e.g. "Owner" showed nothing. Co-Authored-By: Claude Opus 4.8 * feat(frontend): home page toolbar and content-filter revamp - "New" create-menu button (scripts/flows/apps/…) replaces the old Content button; the search bar moves to the right of the toggle group. - Restore the content filter dropped in a merge: a `content` searchbar filter swaps the list for the full-text ContentSearchInner view (EE), aligned flush with -mx-2. - Move the owner/group and label chips off the page into FilterSearchbar presets; ownerFilter/labelFilter now derive from the searchbar keys (data layer unchanged). - Move the list controls (select / tree view / expand-all / sort) inline into the top row between the toggle group and search bar; add margin above the list. - Beta tag on the home AI chat; a bit more bottom margin under it; tighten the gap between the admin/tutorial banners and the list. Co-Authored-By: Claude Opus 4.8 * fix(ai): pass the request body to the free-tier reservation Thread the prompt body into resolve_free_tier_credentials so the free tier can size its upfront reservation from the actual request length instead of a fixed worst case (EE c2e248b), fixing normal chats being rejected as "too large". Updates the OSS stub signature and bumps ee-repo-ref. Co-Authored-By: Claude Opus 4.8 * fix(frontend): gate home Create/Import menu on edit permissions The relocated CreateActionsMenu rendered unconditionally, so operators and users in workspaces protected from direct deployment saw create/import actions they can't use. Restore the original gate (!operator && showEditButtons, the latter from NoDirectDeployAlert). Co-Authored-By: Claude Opus 4.8 * fix(frontend): address Codex review on filter searchbar - P1: the boolean shortcut now goes through the same tag-insertion path as the normal branch, so it removes the typed search segment instead of leaving it as a stray free-text (_default_) term. - Mark the Runs `show_future_jobs` filter default: true so selecting it opens the picker (false is the meaningful choice) rather than being a no-op. - Home owner/label presets now emit the canonical `key:\ value` form so the applied-preset check matches after a reparse and can't re-offer a duplicate; update the suggestion extraction to strip the leading separator. - Replace deprecated Button props (size/spacingSize/color) on the relocated list controls with unifiedSize. - Fix stale comments: UsageMeter no longer claims a free-tier consumer; the home filter schema comment describes presets, not the removed ListFilters/label badges. Co-Authored-By: Claude Opus 4.8 * fix(frontend): boolean filter shortcut sets value canonically The round-1 shortcut baked `true` into the tag text, which merged into a following tag (e.g. `archived:\ truekind:\ flow`). Instead remove the typed segment, set the value, and reparse so the text is rebuilt canonically — no lingering free-text and no merge. Co-Authored-By: Claude Opus 4.8 * docs(ai): restate free-tier caller identity contract in the OSS stub; bump ee-repo-ref Co-Authored-By: Claude Opus 4.8 * fix(frontend): keep flanking tags separate when boolean shortcut drops a segment Joining `before`/`after` directly fused the tags a removed mid-segment sat between (e.g. `kind:\ flowsummary:\ bar`). Join with a space; reparse then canonicalizes. Also trims the comment to the essential constraint. Co-Authored-By: Claude Opus 4.8 * chore(ai): update sqlx cache for free-tier daily-day queries; bump ee-repo-ref The reserve/reconcile daily-usage queries now bind the reservation day (EE change); refresh their offline query cache and point ee-repo-ref at the EE commit. Co-Authored-By: Claude Opus 4.8 * fix(ai): activate free tier when instance ai_config has no provider An instance ai_config row won precedence just by existing, so an empty {} (valid via global settings / declarative config) suppressed the free-tier fallback and left AI disabled — even though build_copilot_settings_state already treats it as unconfigured. Apply the same has_providers() check to the instance config in the proxy and edit_copilot_config paths. Also refresh the sqlx cache for the reservation ceiling change and bump ee-repo-ref. Co-Authored-By: Claude Opus 4.8 * fix(frontend): migrate legacy Home filter URLs to the searchbar keys The old Home UI stored free-text in `search`, owner scope in `filter`, and could write `kind=all`; the generic searchbar sync uses `_default_`, `owner`, and a kind enum without `all`. Rewrite those params once before the sync reads the URL so shared/bookmarked links restore, and drop `kind=all` which would otherwise wedge later filter edits. Co-Authored-By: Claude Opus 4.8 * fix(ai): empty instance config in get_copilot_info; label user-disabled Home AI - get_copilot_info returned any existing instance ai_config row before the free-tier fallback, so an empty {} disabled AI in the copilot-info UI even though the proxy now serves the free tier. Apply the same has_providers() gate here. - The Home chat overlay said "No AI provider is configured" when the user had disabled AI in account settings (providers still present). Distinguish that state ("Windmill AI is disabled in your account settings") as the docked chat does, and drop the misleading workspace-config button in that case. Co-Authored-By: Claude Opus 4.8 * chore(ai): drop redundant proxy service-account check; trim TreeView comment The service-account exclusion now lives in the free-tier helper, so the proxy calls it directly. Also condense the tree-view resize-transition comment to the essential reason. Bumps ee-repo-ref. Co-Authored-By: Claude Opus 4.8 * docs(frontend): the Home content filter is not EE-gated ContentSearchInner loads the workspace's scripts/flows/apps/resources and matches their contents client-side, so it works on any instance. Drop the misleading "(EE)" from the filter label and the "EE indexer / off-EE fallback" comments. Co-Authored-By: Claude Opus 4.8 * chore(ee): bump ee-repo-ref for free-tier pricing + exhaustion fixes Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): show disabled Home AI overlay statically, not on hover The disabled-state overlay (reason + configure/add-key action) was opacity-0 and pointer-events-none until group-hover, so keyboard and touch users saw an inert composer with no visible remedy. Render it and the composer blur statically when disabled instead. Also bumps ee-repo-ref for the trimmed free-tier comments. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): give account-disabled Home AI overlay a recovery action The account-disabled branch showed a reason but hid every action, on the mistaken premise that account settings has no linkable route. It opens from the #user-settings hash (the same one the sidebar Account menu uses), so link there. Bumps ee-repo-ref for the free-tier fixes. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): gate Home AI composer for operators; a11y and filter-sync fixes - Home composer now uses prefersSessionHandoff($userStore?.operator) instead of isGlobalAiEnabled(): operators reached this route and could submit a prompt into a /sessions page that refuses them, silently dropping it. Also drops the leftover empty header spacer div above the chat. - HomeAIChat: mark the blurred/disabled subtrees inert so keyboard users can't tab into the unreadable textarea (pointer-events-none didn't stop Tab). - ItemsList: keep the role-dependent searchbar keys (include_library, only_user_folders) in the schema unconditionally and toggle `hidden` instead, so useUrlSyncedFilterInstance (which snapshots the key set once) still URL-syncs a key that first appears after a workspace switch. - Bumps ee-repo-ref for the indexer non-parquet build fix. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): keep CLI/MCP connect row for operators; trim filter comment The previous commit gated all of HomeAIChat behind the operator/session check, which also removed the AI-independent CLI/MCP "Connect workspace" drawer that operators (and the sessions-beta opt-out) had on main. Render HomeAIChat for the same audience as before (isGlobalAiEnabled) and gate only the composer (title, input, examples, overlay) on operator status inside the component; the connect row always shows. Also trims the role-dependent filter-schema comment to the <=4 line rule. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): reconnect Home keyboard navigation to the unified searchbar The searchbar migration replaced the the ItemsList keyboard handler keys off, so Arrow/Enter no longer drove the results list. Thread an `id` down to the searchbar's contenteditable (via TaggedTextInput/FilterSearchbar `inputId`) so the handler and the workspace-switch focus restoration find it again; read the caret through the Selection API instead of an 's selectionStart/End; and stand the list's arrows down while the searchbar's suggestion dropdown is open (tracked via onDropdownVisibleChange). In free-text mode the searchbar no longer opens its dropdown on a bare arrow key, so an empty box passes Arrow/Enter to the list as before. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * fix(frontend): stop searchbar Enter inserting a newline; idle typewriter for operators - TaggedTextInput is a single-line filter input, so Enter now preventDefaults the contenteditable's newline insertion (surrounding suggestion-select / list-open handlers still run on bubble). Previously Enter with no row highlighted dropped a literal \n into the query. - HomeAIChat's placeholder typewriter effect now runs only while the composer is shown, so it no longer loops forever driving an unrendered input for operators. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS * chore: update ee-repo-ref to f2a31156ac08ecb02d89dbc66d72be58e9c877ff This commit updates the EE repository reference after PR #652 was merged in windmill-ee-private. Previous ee-repo-ref: e59b96a2eea5d1110b40c842f17b337ab051bdd3 New ee-repo-ref: f2a31156ac08ecb02d89dbc66d72be58e9c877ff Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- ...64fc5e65995ce81e622c174a89befc1a527e5.json | 22 + ...23f37b8d452ea8e199ccc207da238368f1996.json | 23 + ...e6be4cc3adc6aa859c3dbaa57fa50b63741cf.json | 15 + ...fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json | 23 + ...7e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json | 15 + backend/ee-repo-ref.txt | 2 +- ...0260622072905_ai_free_token_usage.down.sql | 2 + .../20260622072905_ai_free_token_usage.up.sql | 19 + backend/summarized_schema.txt | 2 + backend/windmill-api/openapi.yaml | 18 + backend/windmill-api/src/ai.rs | 279 +++++---- backend/windmill-api/src/ai_free_tier_oss.rs | 65 +++ backend/windmill-api/src/lib.rs | 3 + backend/windmill-api/src/workspaces.rs | 58 +- frontend/src/lib/aiStore.ts | 13 +- .../src/lib/components/FilterSearchbar.svelte | 145 ++++- .../src/lib/components/TaggedTextInput.svelte | 11 +- .../lib/components/common/table/AppRow.svelte | 2 +- .../components/common/table/FlowRow.svelte | 2 +- .../components/common/table/RawAppRow.svelte | 2 +- .../lib/components/common/table/Row.svelte | 2 + .../components/common/table/ScriptRow.svelte | 6 +- .../lib/components/copilot/chat/AIChat.svelte | 28 +- .../copilot/chat/AIChatDisplay.svelte | 63 +++ .../copilot/chat/AIChatManager.svelte.ts | 30 +- .../copilot/chat/AIChatModelSettings.svelte | 14 + .../copilot/chat/ContextUsageIndicator.svelte | 28 +- .../components/copilot/chat/UsageMeter.svelte | 39 ++ .../src/lib/components/copilot/loadCopilot.ts | 9 + .../src/lib/components/home/HomeAIChat.svelte | 255 +++++++++ .../src/lib/components/home/ItemsList.svelte | 529 ++++++++++-------- .../lib/components/home/NoItemFound.svelte | 1 - .../src/lib/components/home/TreeView.svelte | 205 +++---- .../src/lib/components/runs/runsFilter.ts | 4 +- .../components/select/GenericDropdown.svelte | 15 +- .../sessions/sessionSwitch.svelte.ts | 6 +- .../src/routes/(root)/(logged)/+page.svelte | 67 +-- 37 files changed, 1457 insertions(+), 565 deletions(-) create mode 100644 backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json create mode 100644 backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json create mode 100644 backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json create mode 100644 backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json create mode 100644 backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json create mode 100644 backend/migrations/20260622072905_ai_free_token_usage.down.sql create mode 100644 backend/migrations/20260622072905_ai_free_token_usage.up.sql create mode 100644 backend/windmill-api/src/ai_free_tier_oss.rs create mode 100644 frontend/src/lib/components/copilot/chat/UsageMeter.svelte create mode 100644 frontend/src/lib/components/home/HomeAIChat.svelte diff --git a/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json new file mode 100644 index 0000000000..4ae74ff53c --- /dev/null +++ b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT cost_nanos FROM ai_free_token_usage WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5" +} diff --git a/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json new file mode 100644 index 0000000000..2253944269 --- /dev/null +++ b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, $1::bigint, now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = ai_free_token_daily_usage.cost_nanos + $1::bigint,\n updated_at = now()\n RETURNING cost_nanos", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Date" + ] + }, + "nullable": [ + false + ] + }, + "hash": "44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996" +} diff --git a/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json new file mode 100644 index 0000000000..a741ee1c0d --- /dev/null +++ b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, GREATEST(0, $1::bigint), now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_daily_usage.cost_nanos + $1::bigint),\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Date" + ] + }, + "nullable": [] + }, + "hash": "acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf" +} diff --git a/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json new file mode 100644 index 0000000000..7078c32f1a --- /dev/null +++ b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, $2::bigint, now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = ai_free_token_usage.cost_nanos + $2::bigint,\n updated_at = now()\n RETURNING cost_nanos", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956" +} diff --git a/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json new file mode 100644 index 0000000000..32f7e8cf1f --- /dev/null +++ b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, GREATEST(0, $2::bigint), now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_usage.cost_nanos + $2::bigint),\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 51788b0e1d..2bb93f6bd9 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -df60763d1f243b0048dfc3fe700bc026b257bea8 +f2a31156ac08ecb02d89dbc66d72be58e9c877ff diff --git a/backend/migrations/20260622072905_ai_free_token_usage.down.sql b/backend/migrations/20260622072905_ai_free_token_usage.down.sql new file mode 100644 index 0000000000..95879034e4 --- /dev/null +++ b/backend/migrations/20260622072905_ai_free_token_usage.down.sql @@ -0,0 +1,2 @@ +DROP TABLE ai_free_token_daily_usage; +DROP TABLE ai_free_token_usage; diff --git a/backend/migrations/20260622072905_ai_free_token_usage.up.sql b/backend/migrations/20260622072905_ai_free_token_usage.up.sql new file mode 100644 index 0000000000..ae0cf47e45 --- /dev/null +++ b/backend/migrations/20260622072905_ai_free_token_usage.up.sql @@ -0,0 +1,19 @@ +-- One-time grant of the Windmill-provided free AI tier, measured as cost in nano-dollars +-- (1e-9 USD) rather than raw tokens — a prompt-cache hit costs a fraction of a fresh input +-- token, so a token count wildly overstates the real bill. The grant never resets: once +-- spent, the user must bring their own API key. Keyed by normalized email so the allowance +-- is shared across a user's workspaces (and is resistant to +tag / gmail-dot aliasing). +CREATE TABLE ai_free_token_usage ( + email VARCHAR(255) PRIMARY KEY, + cost_nanos BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Instance-wide daily cost ceiling (nano-dollars) for the free tier — a kill-switch +-- independent of the per-user grant, bounding the blast radius of a bad day. One row per +-- UTC day. +CREATE TABLE ai_free_token_daily_usage ( + day DATE PRIMARY KEY, + cost_nanos BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 607ef99e0c..dfd0305e57 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -38,6 +38,8 @@ account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), clien FK: (workspace_id) -> workspace(id) agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char) ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) +ai_free_token_daily_usage: day(date), cost_nanos(bigint), updated_at(ts) +ai_free_token_usage: email(char), cost_nanos(bigint), updated_at(ts) ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 09a3cf4195..31f0045ce0 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -27256,11 +27256,29 @@ components: type: integer minimum: 1 maximum: 2000000 + free_tier: + $ref: "#/components/schemas/FreeTierInfo" model_pricing: type: object additionalProperties: $ref: "#/components/schemas/ModelPriceOverride" + FreeTierInfo: + type: object + description: >- + Read-only. Present when the workspace has no AI provider of its own and is running + on Windmill's free tier. Ignored on write. + properties: + exhausted: + type: boolean + description: The one-time grant is spent; no provider is served and the user must add their own API key. + used_ratio: + type: number + description: Fraction of the grant consumed, 0 to 1. + required: + - exhausted + - used_ratio + ModelPriceOverride: type: object description: negotiated rates in USD per million tokens, keyed `provider:model` diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 1a5f9dba87..c684830b27 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -409,6 +409,19 @@ impl ExpiringProviderCredentials { } } +/// Set on the copilot config when the workspace has no AI provider of its own and is +/// running on Windmill's free tier, so the client can label the lent model as free, warn +/// before the grant runs out, and tell the user to add their own key once it has — rather +/// than showing the same "no provider configured" state a never-configured workspace gets. +#[derive(Serialize, Deserialize, Debug, Default, Clone)] +pub struct FreeTierInfo { + /// The grant is spent: no provider is served and the user must bring their own key. + pub exhausted: bool, + /// Fraction of the grant consumed, 0.0..=1.0. A ratio, not a dollar amount — the + /// pricing model stays server-side. + pub used_ratio: f64, +} + #[derive(Serialize, Deserialize, Debug, Default)] pub struct AIConfig { #[serde(skip_serializing_if = "Option::is_none")] @@ -423,6 +436,11 @@ pub struct AIConfig { pub custom_prompts: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub max_tokens_per_model: Option>, + /// Response-only: this same struct is the request body for saving a workspace's AI + /// config, and `skip_deserializing` is what stops a client from storing a forged + /// free-tier marker. Only the server sets it, per-request. + #[serde(skip_serializing_if = "Option::is_none", skip_deserializing)] + pub free_tier: Option, /// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`. /// Only models whose rates differ from the built-in table are stored. #[serde(skip_serializing_if = "Option::is_none")] @@ -1013,83 +1031,119 @@ async fn proxy( check_scopes(&authed, || format!("resources:read:{}", resource_path))?; } - let mut credentials = match workspace_cache { - Some(request_cache) if !request_cache.is_expired() && forced_resource_path.is_none() => { - request_cache.credentials - } - _ => { - let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) = - if let Some(resource_path) = forced_resource_path { - // forced resource path - (resource_path, false, w_id.clone(), None) - } else { - let workspace_ai_config = sqlx::query_scalar!( - "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; + // Set when serving the request through Windmill's free AI tier (the lent key). Holds + // the per-user concurrency lock and drives response metering. + let mut free_lease: Option = None; + let mut credentials = 'cred: { + match workspace_cache { + Some(request_cache) + if !request_cache.is_expired() && forced_resource_path.is_none() => + { + request_cache.credentials + } + _ => { + let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) = + if let Some(resource_path) = forced_resource_path { + // forced resource path + (resource_path, false, w_id.clone(), None) + } else { + let workspace_ai_config = sqlx::query_scalar!( + "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; - let (ai_config_value, resource_workspace, instance_ai_config_revision) = { - let ws_has_config = workspace_ai_config - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .is_some_and(|config| config.has_providers()); + let (ai_config_value, resource_workspace, instance_ai_config_revision) = { + let ws_has_config = workspace_ai_config + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .is_some_and(|config| config.has_providers()); - if ws_has_config { - (workspace_ai_config.unwrap(), w_id.clone(), None) - } else { - let instance_config = sqlx::query_scalar!( - "SELECT value FROM global_settings WHERE name = 'ai_config'" - ) - .fetch_optional(&db) - .await?; + if ws_has_config { + (workspace_ai_config.unwrap(), w_id.clone(), None) + } else { + let instance_config = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = 'ai_config'" + ) + .fetch_optional(&db) + .await?; - match instance_config { - Some(config) => ( - config, - "admins".to_string(), - Some(current_instance_ai_config_revision()), - ), - None => { - return Err(Error::internal_err( - "AI resource not configured".to_string(), - )); + let instance_has_config = + instance_config.as_ref().is_some_and(|v| { + serde_json::from_value::(v.clone()) + .ok() + .is_some_and(|c| c.has_providers()) + }); + match instance_config { + // An instance `ai_config` row with no usable provider (e.g. `{}` + // or `{"providers":{}}`) is treated as unconfigured, exactly as + // build_copilot_settings_state does — otherwise its mere presence + // would suppress the free-tier fallback below. + Some(config) if instance_has_config => ( + config, + "admins".to_string(), + Some(current_instance_ai_config_revision()), + ), + _ => { + // Nothing configured: fall back to Windmill's free AI tier + // (EE-only) if a lent key is set and both the user's + // one-time grant and the instance's daily cap have room. + // Errors once the grant is spent, the day is capped, or the + // user already has a request in flight; None otherwise. + // Ineligible identities (e.g. service accounts) are refused + // inside the helper, so every path treats them alike. + let free = + crate::ai_free_tier_oss::resolve_free_tier_credentials( + &provider, + &db, + &ai_path, + &authed.email, + &body, + ) + .await?; + if let Some((free_credentials, lease)) = free { + free_lease = Some(lease); + break 'cred free_credentials; + } + return Err(Error::internal_err( + "AI resource not configured".to_string(), + )); + } } } + }; + + let mut ai_config = serde_json::from_value::(ai_config_value) + .map_err(|e| Error::BadRequest(e.to_string()))?; + + let provider_config = ai_config + .providers + .as_mut() + .and_then(|providers| providers.remove(&provider)) + .ok_or_else(|| { + Error::BadRequest(format!("Provider {:?} not configured", provider)) + })?; + + if provider_config.resource_path.is_empty() { + return Err(Error::BadRequest("Resource path is empty".to_string())); } + + ( + provider_config.resource_path, + true, + resource_workspace, + instance_ai_config_revision, + ) }; - let mut ai_config = serde_json::from_value::(ai_config_value) - .map_err(|e| Error::BadRequest(e.to_string()))?; - - let provider_config = ai_config - .providers - .as_mut() - .and_then(|providers| providers.remove(&provider)) - .ok_or_else(|| { - Error::BadRequest(format!("Provider {:?} not configured", provider)) - })?; - - if provider_config.resource_path.is_empty() { - return Err(Error::BadRequest("Resource path is empty".to_string())); - } - - ( - provider_config.resource_path, - true, - resource_workspace, - instance_ai_config_revision, - ) - }; - - // For user-specified resources, fetch through an RLS-scoped - // connection so PostgreSQL row-level security enforces the same - // folder/group boundaries as the regular resource API. For the - // workspace/instance ai_config path, the resource_path was already - // validated by an admin/devops user when configuring the workspace, - // so the raw pool is used. - let resource = if is_user_specified_resource { + // For user-specified resources, fetch through an RLS-scoped + // connection so PostgreSQL row-level security enforces the same + // folder/group boundaries as the regular resource API. For the + // workspace/instance ai_config path, the resource_path was already + // validated by an admin/devops user when configuring the workspace, + // so the raw pool is used. + let resource = if is_user_specified_resource { let mut tx = user_db.clone().begin(&authed).await?; let res = sqlx::query_scalar::<_, Option>>>( "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", @@ -1112,38 +1166,45 @@ async fn proxy( .ok_or_else(|| Error::NotFound(format!("Could not find the resource {}, update the resource path in the workspace settings", resource_path)))? .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", resource_path)))?; - let resource = serde_json::from_str::(resource.0.get()) - .map_err(|e| Error::BadRequest(e.to_string()))?; + let resource = serde_json::from_str::(resource.0.get()) + .map_err(|e| Error::BadRequest(e.to_string()))?; - // Enforce RLS on $var: resolution when the resource path was - // user-specified (X-Resource-Path header) so users can only read - // variables they have permission to access. - let enforce_authed = if is_user_specified_resource { - Some(&authed) - } else { - None - }; - let credentials = resolve_provider_credentials( - &provider, - &db, - &resource_workspace, - resource, - enforce_authed, - ) - .await?; - if save_to_cache { - AI_REQUEST_CACHE.insert( - (w_id.clone(), provider.clone()), - ExpiringProviderCredentials::new( - credentials.clone(), - instance_ai_config_revision, - ), - ); + // Enforce RLS on $var: resolution when the resource path was + // user-specified (X-Resource-Path header) so users can only read + // variables they have permission to access. + let enforce_authed = if is_user_specified_resource { + Some(&authed) + } else { + None + }; + let credentials = resolve_provider_credentials( + &provider, + &db, + &resource_workspace, + resource, + enforce_authed, + ) + .await?; + if save_to_cache { + AI_REQUEST_CACHE.insert( + (w_id.clone(), provider.clone()), + ExpiringProviderCredentials::new( + credentials.clone(), + instance_ai_config_revision, + ), + ); + } + credentials } - credentials } }; + // Free tier: pin the model and clamp max_tokens server-side before forwarding, + // since the request body is otherwise client-controlled. + if free_lease.is_some() { + body = crate::ai_free_tier_oss::enforce_free_tier_body(&body)?; + } + if let Some(fim_transform) = maybe_transform_fim_request(&provider, &ai_path, &credentials.base_url, &body)? { @@ -1291,8 +1352,32 @@ async fn proxy( let status_code = response.status(); let headers = response.headers().clone(); + let is_sse = is_sse_response(&headers); + + // Free tier: reconcile the cost reserved up-front against what the response actually + // used, holding the per-user lock (via the lease) until it is recorded. The chat + // streams (SSE), where the usage report only arrives in the final chunk; the + // non-streaming JSON path is handled for completeness. + if let Some(lease) = free_lease { + let body = if is_sse { + axum::body::Body::from_stream(inject_keepalives( + Box::pin(crate::ai_free_tier_oss::meter_usage( + response.bytes_stream(), + db.clone(), + lease, + )), + Duration::from_secs(KEEPALIVE_INTERVAL_SECS), + )) + } else { + let bytes = response.bytes().await.map_err(to_anyhow)?; + crate::ai_free_tier_oss::record_json_usage(db.clone(), lease, &bytes); + axum::body::Body::from(bytes) + }; + return Ok((status_code, headers, body)); + } + let stream = response.bytes_stream(); - let body = if is_sse_response(&headers) { + let body = if is_sse { axum::body::Body::from_stream(inject_keepalives( stream, Duration::from_secs(KEEPALIVE_INTERVAL_SECS), diff --git a/backend/windmill-api/src/ai_free_tier_oss.rs b/backend/windmill-api/src/ai_free_tier_oss.rs new file mode 100644 index 0000000000..7a183ff8ed --- /dev/null +++ b/backend/windmill-api/src/ai_free_tier_oss.rs @@ -0,0 +1,65 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ai_free_tier_ee::*; + +// Open-source build: Windmill's free AI tier does not exist. These stubs make the +// callers in `ai.rs` / `workspaces.rs` compile while disabling the feature entirely — +// `resolve_free_tier_credentials` never opts in, so the proxy falls through to its +// normal "AI resource not configured" path and the copilot stays hidden. +// +// Caller contract (enforced by the private impl, restated here for parity): the `email` +// passed to `resolve_free_tier_credentials` / `free_tier_copilot_config` MUST be the +// authenticated caller's own identity (an `ApiAuthed` email), never a client-supplied one — +// it selects whose lent-key grant is spent and whose usage is read. + +#[cfg(not(feature = "private"))] +use crate::ai::AIConfig; +#[cfg(not(feature = "private"))] +use crate::db::DB; +#[cfg(not(feature = "private"))] +use axum::body::Bytes; +#[cfg(not(feature = "private"))] +use windmill_ai::ai_providers::AIProvider; +#[cfg(not(feature = "private"))] +use windmill_ai::credentials::ProviderCredentials; +#[cfg(not(feature = "private"))] +use windmill_common::error::Result; + +#[cfg(not(feature = "private"))] +pub struct FreeTierLease; + +#[cfg(not(feature = "private"))] +pub async fn resolve_free_tier_credentials( + _provider: &AIProvider, + _db: &DB, + _ai_path: &str, + _email: &str, + _body: &Bytes, +) -> Result> { + Ok(None) +} + +#[cfg(not(feature = "private"))] +pub fn enforce_free_tier_body(body: &Bytes) -> Result { + Ok(body.clone()) +} + +#[cfg(not(feature = "private"))] +pub async fn free_tier_copilot_config(_db: &DB, _email: &str) -> Result> { + Ok(None) +} + +#[cfg(not(feature = "private"))] +pub fn record_json_usage(_db: DB, _lease: FreeTierLease, _bytes: &[u8]) {} + +#[cfg(not(feature = "private"))] +pub fn meter_usage( + upstream: S, + _db: DB, + _lease: FreeTierLease, +) -> impl futures::Stream> +where + S: futures::Stream> + Unpin, +{ + upstream +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index de49949043..fc98020d1b 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -66,6 +66,9 @@ use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; mod ai; +#[cfg(feature = "private")] +mod ai_free_tier_ee; +mod ai_free_tier_oss; mod ai_skills; mod apps; mod apps_raw_bundle; diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 01eb32b7f6..76378e335a 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -18,7 +18,6 @@ use crate::teams_oss::{ connect_teams, edit_teams_command, run_teams_message_test_job, workspaces_list_available_teams_channels, workspaces_list_available_teams_ids, }; - use axum::{ extract::{Extension, Path}, routing::{get, post}, @@ -153,10 +152,23 @@ async fn edit_copilot_config( .await?; let settings_state = build_copilot_settings_state(workspace_has_config, instance_ai_config.as_ref()); + // A provider-less instance config (e.g. `{}`) is unconfigured, same as build_copilot_settings_state + // treats it — so it must not shadow the free-tier fallback here either. + let instance_config_with_providers = instance_ai_config + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .filter(|c| c.has_providers()); let effective_ai_config = if workspace_has_config { ai_config - } else if let Some(instance_ai_config) = instance_ai_config { - serde_json::from_value::(instance_ai_config).unwrap_or_default() + } else if let Some(instance_config) = instance_config_with_providers { + instance_config + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? + { + // Same fallback as get_copilot_info: with nothing configured, surface Windmill's free + // tier (EE-only) so clearing a workspace provider activates it immediately, instead of + // returning an empty config that disables AI until the next page reload re-fetches it. + free_config } else { AIConfig::default() }; @@ -179,6 +191,7 @@ struct EditCopilotConfigResponse { } async fn get_copilot_info( + authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, ) -> JsonResult { @@ -194,16 +207,25 @@ async fn get_copilot_info( )) })?; - if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { - Ok(Json(workspace_ai_config.0)) - } else if let Some(instance_config) = + let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) .await? + .and_then(|v| serde_json::from_value::(v).ok()) + // A provider-less instance config (e.g. `{}`) is unconfigured; don't let it shadow the + // free-tier fallback, matching the proxy and edit_copilot_config paths. + .filter(|c| c.has_providers()); + if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { + Ok(Json(workspace_ai_config.0)) + } else if let Some(instance_config) = instance_config { + Ok(Json(instance_config)) + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? { - Ok(Json( - serde_json::from_value::(instance_config).unwrap_or_default(), - )) + // Nothing configured: fall back to Windmill's free tier (EE-only). The config + // carries a `free_tier` marker even once the user's grant is spent — with no + // providers, but telling the client *why* AI is off. + Ok(Json(free_config)) } else { Ok(Json(AIConfig::default())) } @@ -216,7 +238,14 @@ pub async fn get_critical_alerts( authed: ApiAuthed, Query(params): Query, ) -> JsonResult { - require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, authed.job_id.is_some(), &db).await?; + require_admin_or_devops( + authed.is_admin, + &authed.username, + &authed.email, + authed.job_id.is_some(), + &db, + ) + .await?; crate::utils::get_critical_alerts(db, params, Some(w_id)).await } @@ -232,7 +261,14 @@ pub async fn acknowledge_critical_alert( Path((w_id, id)): Path<(String, i32)>, authed: ApiAuthed, ) -> Result { - require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, authed.job_id.is_some(), &db).await?; + require_admin_or_devops( + authed.is_admin, + &authed.username, + &authed.email, + authed.job_id.is_some(), + &db, + ) + .await?; crate::utils::acknowledge_critical_alert(db, Some(w_id), id).await } diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 0955f68ee5..2d6a98c625 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -7,6 +7,7 @@ import { type AIProviderModel, type AIProvider, type AIConfig, + type FreeTierInfo, type ModelPriceOverride } from './gen' import { @@ -49,6 +50,10 @@ export const copilotInfo = writable<{ /** Negotiated rates per `provider:model`, overriding the built-in price table. */ modelPricing?: Record webSearchEnabledProviders?: Partial> + // Set only when the workspace has no AI provider of its own and is running on + // Windmill's free tier. `exhausted` means the grant is spent: there is no model, but + // that is a different state from "never configured" and the UI must say so. + freeTier?: FreeTierInfo }>({ enabled: false, codeCompletionModel: undefined, @@ -132,8 +137,9 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, + webSearchEnabledProviders, modelPricing: aiConfig.model_pricing ?? {}, - webSearchEnabledProviders + freeTier: aiConfig.free_tier }) } else { copilotSessionModel.set(undefined) @@ -146,8 +152,11 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + webSearchEnabledProviders: {}, modelPricing: {}, - webSearchEnabledProviders: {} + // An exhausted free grant lands here — no providers, but the reason AI is off + // is "you used it up", not "you never set it up". + freeTier: aiConfig.free_tier }) } } diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index 5078e0a83b..51026f10fd 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -9,6 +9,11 @@ type: 'string' | 'number' | 'boolean' allowMultiple?: boolean format?: 'json' + /** Boolean only: the value the filter holds while unset (defaults to false). + * Selecting a filter whose default is false sets it to true immediately rather + * than opening a true/false picker whose only useful choice is true. A default-true + * boolean still opens the picker, since choosing false is the meaningful action. */ + default?: boolean } | { type: 'date' @@ -121,16 +126,34 @@ // Create the filter instance object const filterInstance: { val: Partial> } = $state({ val: {} }) - // Sync URL params to filter instance on initialization and when URL changes + // Sync URL params to filter instance, reactively. Reading urlFilter[key] tracked + // means browser Back/Forward — which mutates useSearchParams' cells on popstate — + // flows into the instance (chips, kind toggle, results), not just the first render. + // The write happens untracked so it can't self-trigger, and the equality check plus + // the reverse effect's own guard keep the two directions from ping-ponging. for (const key of Object.keys(schemaRec)) { - let urlValue = urlFilter[key] - if (schemaRec[key].type === 'date' && typeof urlValue === 'string') { - const d = new Date(urlValue) - urlValue = isNaN(d.getTime()) ? null : d - } - if (urlValue !== undefined && urlValue !== null) { - ;(filterInstance.val as any)[key] = urlValue - } + $effect(() => { + let urlValue = urlFilter[key] + if (schemaRec[key].type === 'date' && typeof urlValue === 'string') { + const d = new Date(urlValue) + urlValue = isNaN(d.getTime()) ? null : d + } + untrack(() => { + const current = (filterInstance.val as any)[key] + const same = + urlValue instanceof Date && current instanceof Date + ? urlValue.getTime() === current.getTime() + : current === (urlValue ?? undefined) + if (same) return + if (urlValue !== undefined && urlValue !== null) { + ;(filterInstance.val as any)[key] = urlValue + } else if (current !== undefined) { + // Key dropped from the URL (Back to a state without it): clear it so a + // stale chip / filter doesn't linger against the navigated-to URL. + delete (filterInstance.val as any)[key] + } + }) + }) } // Sync filter instance changes back to URL params @@ -275,6 +298,17 @@ class?: string placeholder?: string autofocus?: boolean + // Applied as the id of the underlying editable, so a parent can focus it or recognise its + // key events by id (the searchbar is a contenteditable, not an ). + inputId?: string + // Free-text mode: while the input holds only free text (no specific filter tag is + // being edited and no non-default filter is set), suppress the suggestions dropdown + // so it behaves like a plain search box. This frees the arrow keys for the + // surrounding UI (e.g. a results list). The dropdown returns the moment a specific + // filter is present (e.g. `path: u/me/abc`). + hideDropdownOnFreeText?: boolean + // Notified whenever the dropdown's effective visibility changes + onDropdownVisibleChange?: (visible: boolean) => void } type SchemaT = FilterSchemaRec // TODO: Generic @@ -284,7 +318,10 @@ presets: _presets = [], class: className, placeholder = 'Filter...', - autofocus + autofocus, + hideDropdownOnFreeText = false, + onDropdownVisibleChange, + inputId }: Props = $props() let _value = new DebouncedTempValue( @@ -298,6 +335,24 @@ let currentTag: keyof SchemaT | undefined = $state() let currentTextSegment = $state({ text: '', start: 0, end: 0 }) let open = $state(false) + + // A specific filter is in play when a tag is being edited or any non-free-text filter + // is set. + let hasSpecificFilter = $derived( + !!currentTag || Object.keys(value).some((k) => k !== '_default_') + ) + // A plain search term is being typed (free text, no specific filter). + let hasFreeText = $derived(!!String(value['_default_'] ?? '').trim()) + // Effective dropdown visibility. Free-text mode suppresses the dropdown ONLY while the + // user is typing a plain search term: it still opens when the input is empty (so the + // available filters stay discoverable) and whenever a specific filter is set or being + // edited. That leaves the arrow keys for the surrounding list only during free-text search. + let dropdownVisible = $derived( + open && (!hideDropdownOnFreeText || hasSpecificFilter || !hasFreeText) + ) + $effect(() => { + onDropdownVisibleChange?.(dropdownVisible) + }) let inputElement: HTMLDivElement | undefined = $state() let highlightedIndex = $state(0) let taggedTextInput: TaggedTextInput | undefined = $state() @@ -347,9 +402,17 @@ key, filterSchema, onClick: () => { - // Replace the text segment with the new filter tag const before = asText.val.slice(0, currentTextSegment.start) const after = asText.val.slice(currentTextSegment.end) + if (schema[key].type === 'boolean' && schema[key].default !== true) { + // Set the only useful value and reparse to canonical text. The space is + // required: dropping the segment must not fuse the tags that flanked it. + asText.val = `${before} ${after}` + value[key] = true as any + asText.reparse() + return + } + // Replace the text segment with the new (empty) filter tag; the value picker opens. asText.val = `${before}${before && !before.endsWith(' ') ? ' ' : ''}${key}:\\\u00A0${after}`.trim() + '\u00A0' @@ -406,6 +469,28 @@ onClick: () => setValueForCurrentTag(false) } ] + } else if (filter.type === 'string' && filter.format !== 'json') { + // A plain string filter has no fixed options, but any presets targeting this tag + // (`:`) are exactly its useful values — surface them as suggestions so + // picking one is a click, matching the top-level preset row. Unescape the tagged + // syntax's `\ ` back to a real space for the stored value. + const prefix = `${String(currentTag)}:` + const suffix = String(value[currentTag!] ?? '') + .trim() + .toLowerCase() + return _presets + .filter((p) => p.value.startsWith(prefix) && !asText.val.includes(p.value)) + .map((p) => { + const raw = p.value.slice(prefix.length).replace(/^\\ /, '').replace(/\\ /g, ' ') + return { name: p.name, raw } + }) + .filter((p) => !suffix || p.raw.toLowerCase().includes(suffix)) + .map((p) => ({ + type: 'option' as const, + option: { value: p.raw, label: p.name }, + onClick: () => appendOrSetValueForCurrentTag(p.raw), + onNegativeClick: undefined + })) } } return [] @@ -514,7 +599,9 @@ } function handleKeyDown(e: KeyboardEvent) { - if (!open) return + // In free-text mode the dropdown is hidden; let arrow/enter keys pass through to + // the surrounding UI (e.g. list navigation) rather than steering a hidden menu. + if (!dropdownVisible) return if (e.key === 'Escape') { open = false return @@ -601,6 +688,7 @@ > (open = true)} + onKeyDown={(e) => { + // In free-text mode the searchbar coexists with a list that owns Arrow/Enter, so opening + // the dropdown on a bare navigation key would steal them from an empty box. Typing, click, + // or an already-open dropdown still open/keep it. Other searchbars keep opening on any key. + if ( + !hideDropdownOnFreeText || + !['ArrowDown', 'ArrowUp', 'ArrowLeft', 'ArrowRight', 'Enter', 'Escape', 'Tab'].includes( + e.key + ) + ) { + open = true + } + }} {autofocus} /> {#if asText.val} @@ -630,9 +730,10 @@
    inputElement?.getBoundingClientRect() ?? new DOMRect()} - innerClass="!max-h-[30rem]" + innerClass="!max-h-[25rem]" strictWidth > @@ -747,6 +848,20 @@ class="border border-border-light rounded min-h-[4rem]" />
    + {:else if filter.type === 'string'} + {#if menuItems.length} +
    + {#each menuItems as item, index} + {#if item.type === 'option' && item.option} + {@render menuItem({ + onClick: item.onClick, + label: item.option.label || item.option.value, + highlighted: index === highlightedIndex + })} + {/if} + {/each} +
    + {/if} {/if} {/snippet} diff --git a/frontend/src/lib/components/TaggedTextInput.svelte b/frontend/src/lib/components/TaggedTextInput.svelte index 23bbc65e6b..9f19195b85 100644 --- a/frontend/src/lib/components/TaggedTextInput.svelte +++ b/frontend/src/lib/components/TaggedTextInput.svelte @@ -8,6 +8,7 @@ onTextSegmentAtCursorChange, onKeyDown, autofocus, + id, class: className = '' }: { tags: { regex: RegExp; id: string; onClear?: () => void }[] @@ -18,6 +19,7 @@ onTextSegmentAtCursorChange?: (segment: { text: string; start: number; end: number }) => void onKeyDown?: (e: KeyboardEvent) => void autofocus?: boolean + id?: string class?: string } = $props() @@ -337,7 +339,13 @@ function handleKeyDown(e: KeyboardEvent) { onKeyDown?.(e) - if (e.key === 'ArrowDown' || e.key === 'ArrowUp' || e.key === 'Enter') return + // Single-line filter input: block Enter's default newline insertion. Surrounding handlers + // (suggestion select, list open) still run on bubble; only the contenteditable break is gone. + if (e.key === 'Enter') { + e.preventDefault() + return + } + if (e.key === 'ArrowDown' || e.key === 'ArrowUp') return const cursorPos = getCursorPosition() const text = getTextContent() @@ -509,6 +517,7 @@
    {#snippet badges()} diff --git a/frontend/src/lib/components/common/table/FlowRow.svelte b/frontend/src/lib/components/common/table/FlowRow.svelte index a155ca3a97..8d88fe8f01 100644 --- a/frontend/src/lib/components/common/table/FlowRow.svelte +++ b/frontend/src/lib/components/common/table/FlowRow.svelte @@ -132,13 +132,13 @@ : `${base}/flows/get/${flow.path}?workspace=${$workspaceStore}`} kind="flow" workspaceId={flow.workspace_id ?? $workspaceStore ?? ''} + {keyboardSelected} {marked} path={flow.draft_path ?? flow.path} summary={flow.is_draft ? `${flow.summary || flow.draft_path || flow.path}*` : flow.summary} {errorHandlerMuted} canFavorite={!flow.draft_only} {depth} - {keyboardSelected} {rowSelection} > {#snippet badges()} diff --git a/frontend/src/lib/components/common/table/RawAppRow.svelte b/frontend/src/lib/components/common/table/RawAppRow.svelte index 3f74356506..ff31eb1aea 100644 --- a/frontend/src/lib/components/common/table/RawAppRow.svelte +++ b/frontend/src/lib/components/common/table/RawAppRow.svelte @@ -35,13 +35,13 @@ {#snippet badges()} diff --git a/frontend/src/lib/components/common/table/Row.svelte b/frontend/src/lib/components/common/table/Row.svelte index 849d77e2d9..77434d3c59 100644 --- a/frontend/src/lib/components/common/table/Row.svelte +++ b/frontend/src/lib/components/common/table/Row.svelte @@ -13,6 +13,8 @@ interface Props { marked: string | undefined selected?: boolean + /** Highlighted by the list's keyboard arrow-navigation (distinct from `selected`, + * which is the checkbox multi-select state). Scrolls itself into view. */ keyboardSelected?: boolean disabled?: boolean canFavorite?: boolean diff --git a/frontend/src/lib/components/common/table/ScriptRow.svelte b/frontend/src/lib/components/common/table/ScriptRow.svelte index 25b557690e..de0a327b1b 100644 --- a/frontend/src/lib/components/common/table/ScriptRow.svelte +++ b/frontend/src/lib/components/common/table/ScriptRow.svelte @@ -147,6 +147,7 @@ ? `${base}/scripts/edit/${script.path}` : `${base}/scripts/get/${script.hash}?workspace=${$workspaceStore}`} kind="script" + {keyboardSelected} {marked} path={script.draft_path ?? script.path} summary={script.is_draft @@ -156,7 +157,6 @@ workspaceId={$workspaceStore ?? ''} canFavorite={!script.draft_only} {depth} - {keyboardSelected} {rowSelection} > {#snippet badges()} @@ -187,7 +187,9 @@ CI test {/if} - {#if script.kind !== 'script'} + + {#if script.kind && script.kind !== 'script'} {script.kind === 'failure' ? 'Error handler' : capitalize(script.kind)} diff --git a/frontend/src/lib/components/copilot/chat/AIChat.svelte b/frontend/src/lib/components/copilot/chat/AIChat.svelte index 2a51b1b2e5..e4dba770eb 100644 --- a/frontend/src/lib/components/copilot/chat/AIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChat.svelte @@ -51,20 +51,26 @@ aiChatManager.scriptEditorOptions?.lang && !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang)) ) + // A spent free grant is not an unconfigured workspace: AIChatDisplay already shows an + // in-thread banner naming the real cause and linking to the key settings, so the generic + // "enable Windmill AI" line would both duplicate it and misstate why the chat is off. + const freeTierExhausted = $derived($copilotInfo.freeTier?.exhausted === true) const disabledMessage = $derived( forceDisabled ? forceDisabledMessage - : !hasCopilot - ? $aiUserDisabled - ? 'Windmill AI is disabled in your account settings' - : isAdmin - ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` - : 'Ask an admin to enable Windmill AI in this workspace to use this chat' - : aiChatManager.mode === AIMode.SCRIPT && - aiChatManager.scriptEditorOptions?.lang && - !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) - ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` - : '' + : freeTierExhausted + ? '' + : !hasCopilot + ? $aiUserDisabled + ? 'Windmill AI is disabled in your account settings' + : isAdmin + ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` + : 'Ask an admin to enable Windmill AI in this workspace to use this chat' + : aiChatManager.mode === AIMode.SCRIPT && + aiChatManager.scriptEditorOptions?.lang && + !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) + ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` + : '' ) const suggestions = [ diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index c68945695b..2ab29499ce 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -15,6 +15,7 @@ Folder, Hand, HistoryIcon, + KeyRound, MousePointer2, Plug, Plus, @@ -59,9 +60,22 @@ readDroppedEntries } from './files/fsAccess' import { sendUserToast } from '$lib/toast' + import Alert from '$lib/components/common/alert/Alert.svelte' + import { copilotInfo } from '$lib/aiStore' + import { base } from '$lib/base' const MAX_YOLO_TOOLTIP_TOOLS = 8 const aiChatManager = getAiChatManager() + + // The user spent their one-time free Windmill AI grant: there is no model left to send + // to, so say so in the thread itself rather than only failing on send. + let freeTierExhausted = $derived($copilotInfo.freeTier?.exhausted === true) + // Still on the free grant: keep how much is left in view right above the composer, so + // running out isn't a surprise. Once spent, the exhausted banner replaces it. + let freeTier = $derived($copilotInfo.freeTier) + let freeTierUsedPct = $derived(Math.min(100, Math.round((freeTier?.used_ratio ?? 0) * 100))) + let showFreeTierUsage = $derived(!!freeTier && !freeTier.exhausted) + // One row per autonomy posture, in picker order, so adding one touches only this // table. `isAvailable` hides the postures that would do nothing in the current AI // mode, which is why the picker can be shorter than this list. @@ -562,6 +576,44 @@ ) +{#snippet freeTierExhaustedBanner()} +
    + +
    + + You have used all of your free Windmill AI tokens. Add your own API key to keep using AI. + + +
    +     +
    +{/snippet} + +{#snippet freeTierUsageBanner()} +
    + + {freeTierUsedPct}% of your free Windmill AI used + + +
    +{/snippet} +
    script editor to modify selected lines. {/if} + {#if freeTierExhausted} +
    + {@render freeTierExhaustedBanner()} +
    + {/if} {/if} {#if messages.length > 0} @@ -699,6 +756,9 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> isLast={messageIndex === messages.length - 1} /> {/each} + {#if freeTierExhausted} + {@render freeTierExhaustedBanner()} + {/if} {#if showTypingIndicator}
    {#if inputPreface} {@render inputPreface()} {/if} + {#if showFreeTierUsage} + {@render freeTierUsageBanner()} + {/if} = 80) + let capability = $derived( getReasoningCapability(providerModel.provider as AIProvider, providerModel.model) ) @@ -312,6 +319,13 @@ {#if effortLabel} · {effortLabel} {/if} + {#if freeTier && !freeTier.exhausted} + Free + {/if}
    diff --git a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte index ffa06ca5df..96b452120b 100644 --- a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte @@ -3,7 +3,7 @@ import { getKnownModelContextWindow, getModelContextWindow } from '../modelConfig' import { getAiChatManager } from './aiChatManagerContext' import { AIMode } from './AIChatManager.svelte' - import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' + import UsageMeter from './UsageMeter.svelte' import { formatTokenCount } from './tokenUsage' const aiChatManager = getAiChatManager() @@ -49,25 +49,11 @@ {#if visible} - - -
    -
    -
    -
    -
    - {#snippet text()} + + + {#snippet tooltip()}

    Context usage

    @@ -85,5 +71,5 @@ {/if}

    {/snippet} -     + {/if} diff --git a/frontend/src/lib/components/copilot/chat/UsageMeter.svelte b/frontend/src/lib/components/copilot/chat/UsageMeter.svelte new file mode 100644 index 0000000000..26e4758953 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/UsageMeter.svelte @@ -0,0 +1,39 @@ + + + +
    +
    +
    +
    +
    + {#snippet text()} + {@render tooltip()} + {/snippet} +     diff --git a/frontend/src/lib/components/copilot/loadCopilot.ts b/frontend/src/lib/components/copilot/loadCopilot.ts index 4634833b0b..496fb89fdc 100644 --- a/frontend/src/lib/components/copilot/loadCopilot.ts +++ b/frontend/src/lib/components/copilot/loadCopilot.ts @@ -1,6 +1,14 @@ import { WorkspaceService } from '$lib/gen' import { copilotWorkspace, setCopilotInfo } from '$lib/aiStore' import { workspaceAIClients } from './lib' +import { writable } from 'svelte/store' + +// The workspace of the most recent loadCopilot *request*, set synchronously before the +// await — as opposed to `copilotWorkspace`, which only updates once a load resolves. A +// background refresh (e.g. free-tier usage) compares against this so it can't supersede an +// in-flight load for a newer workspace (which would otherwise win the token and restore +// stale state). +export const copilotWorkspaceRequested = writable(undefined) // Lives here, not in $lib/aiStore, purely so that module needs no import of the AI // client — it is the one thing that wanted both. Moving it back recreates the @@ -20,6 +28,7 @@ let loadCopilotToken = 0 let inFlight: { workspace: string; promise: Promise } | undefined export function loadCopilot(workspace: string): Promise { + copilotWorkspaceRequested.set(workspace) if (inFlight?.workspace === workspace) { return inFlight.promise } diff --git a/frontend/src/lib/components/home/HomeAIChat.svelte b/frontend/src/lib/components/home/HomeAIChat.svelte new file mode 100644 index 0000000000..d0e75538ca --- /dev/null +++ b/frontend/src/lib/components/home/HomeAIChat.svelte @@ -0,0 +1,255 @@ + + + + +
    +
    + {#if showComposer} +
    +
    +

    Build with AI

    + Beta +
    + +
    + + +
    + +
    +
    +
    + {/if} + +
    + {#if showComposer} +
    + {#each homeAIExamples as example (example.label)} + + {/each} +
    + {:else} +
    + {/if} + + +
    + + {#if !$userStore?.operator && HOME_SHOW_HUB} + + {/if} +
    +
    + {#if showComposer && disabled} +
    +

    + {#if $aiUserDisabled} + Windmill AI is disabled in your account settings + {:else if freeTierExhausted} + You have used all of your free Windmill AI tokens + {:else} + No AI provider is configured + {/if} +

    + {#if $aiUserDisabled} + + + {:else} + + {/if} +
    + {/if} +
    +
    + + diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index d7717d1e04..0f0cb6a6c0 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -1,7 +1,7 @@