From fb82f36e6d6492dd0740984d8d78ea4eaa30361e Mon Sep 17 00:00:00 2001
From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com>
Date: Thu, 27 Aug 2026 18:51:08 +0200
Subject: [PATCH 001/310] fix: pre-fill the test panel JSON args editor and
align its placeholder (#10871)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* fix: pre-fill the test panel JSON args editor and align its placeholder
Co-Authored-By: Claude Opus 5 (1M context)
* fix: reseed the JSON args editor when the preprocessor tab is selected
Co-Authored-By: Claude Opus 5 (1M context)
* fix: seed schema defaults and own-property args in the JSON payload
Co-Authored-By: Claude Opus 5 (1M context)
* fix: follow the schema in an untouched JSON payload, ignore same-tab clicks
Co-Authored-By: Claude Opus 5 (1M context)
* fix: latch JSON editor ownership from Monaco, drop the late remounts
The pristine check read the bound `code` value, which trails the buffer by
SimpleEditor's 200ms debounce — a reseed arriving in that window overwrote text
already typed. Latch ownership from Monaco's own change event instead, via a new
undebounced `input` event guarded so `setCode`'s `setValue` does not read as an
edit.
Both `.then(() => argsRender++)` bumps are gone: the arg views now remount at the
tab transition only, and follow the schema in through `initialCode` when
inference resolves, so a remount can no longer land on an in-progress payload.
`FlowPreviewContent.selectInput` overwrote the editor on select but not on
deselect, leaving the abandoned input's payload over reverted args.
Co-Authored-By: Claude Opus 5 (1M context)
---------
Co-authored-by: Claude Opus 5 (1M context)
---
frontend/src/lib/components/Dev.svelte | 7 +-
.../lib/components/FlowPreviewContent.svelte | 15 +++-
frontend/src/lib/components/JsonInputs.svelte | 41 +++++++++--
frontend/src/lib/components/RunForm.svelte | 10 ++-
.../src/lib/components/ScriptEditor.svelte | 72 ++++++++++++++-----
.../src/lib/components/SimpleEditor.svelte | 30 +++++++-
frontend/src/lib/schema.test.ts | 62 ++++++++++++++++
frontend/src/lib/schema.ts | 30 ++++++++
.../(logged)/flows/get/[...path]/+page.svelte | 5 +-
.../scripts/get/[...hash]/+page.svelte | 9 +--
10 files changed, 239 insertions(+), 42 deletions(-)
create mode 100644 frontend/src/lib/schema.test.ts
diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte
index 54a68dd5a3..c7e83b8a80 100644
--- a/frontend/src/lib/components/Dev.svelte
+++ b/frontend/src/lib/components/Dev.svelte
@@ -1,6 +1,7 @@
@@ -154,17 +145,15 @@
{#snippet actions()}
-
-
-
+
{/snippet}
@@ -260,8 +249,8 @@
Developers
- Calculated on the MAXIMUM number of users in a given billing
- period, see the Customer Portal for more info.
+ Calculated on the MAXIMUM number of users in a given billing period, see the
+ Customer Portal for more info.
@@ -276,8 +265,8 @@
Operators
- Calculated on the MAXIMUM number of operators in a given
- billing period, see the Customer Portal for more info.
+ Calculated on the MAXIMUM number of operators in a given billing period, see
+ the Customer Portal for more info.
- One execution equals one job
- up to 1 second on a worker with 2GB of memory, with each additional
- second counting as an extra execution.
+ One execution equals one job up to 1 second on a worker with 2GB of memory,
+ with each additional second counting as an extra execution.
@@ -365,8 +354,8 @@
Used seats (billed)
- Highest between seats from 'Developers + Operators' and 'Seats from executions'.
- This is the number of seats used for billing this month.
+ Highest between seats from 'Developers + Operators' and 'Seats from
+ executions'. This is the number of seats used for billing this month.
u + c = {formatNumber(premiumInfo.usedSeats)}
@@ -398,8 +387,8 @@
Estimate your monthly cost
- This is a rough estimate based on your expected team size and workload. Actual billing is based
- on the maximum number of users and executions in a given month.
+ This is a rough estimate based on your expected team size and workload. Actual billing is
+ based on the maximum number of users and executions in a given month.
- One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, with
- each additional second counting as an extra execution.
+ One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory,
+ with each additional second counting as an extra execution.
Every seat includes 10 000 executions
- One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, with
- each additional second counting as an extra execution.
+ One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory,
+ with each additional second counting as an extra execution.
{:else}
@@ -586,9 +573,7 @@
{/if}
{:else}
-
- Workspace is on the team plan
-
+
Workspace is on the team plan
{/if}
{:else if planTitle == 'Enterprise'}
{#if plan != 'enterprise'}
@@ -601,9 +586,7 @@
See more
{:else}
-
- Workspace is on enterprise plan
-
+
Workspace is on enterprise plan
{/if}
{:else if planTitle === 'Free'}
{#if plan}
@@ -611,9 +594,7 @@
Cancel your plan in the Customer Portal to downgrade to the free plan
{:else}
-
- Workspace is on the free plan
-
+
Workspace is on the free plan
{/if}
{/if}
diff --git a/frontend/src/lib/components/sidebar/SidebarUsage.svelte b/frontend/src/lib/components/sidebar/SidebarUsage.svelte
index e62cd02565..1625ed83e2 100644
--- a/frontend/src/lib/components/sidebar/SidebarUsage.svelte
+++ b/frontend/src/lib/components/sidebar/SidebarUsage.svelte
@@ -2,21 +2,18 @@
import { resource } from 'runed'
import { goto } from '$lib/navigation'
import { isCloudHosted } from '$lib/cloud'
- import { UserService } from '$lib/gen'
+ import { WorkspaceService } from '$lib/gen'
import {
isPremiumStore,
usageStore,
userStore,
- userWorkspaces,
workspaceMembershipVersion,
workspaceStore,
- workspaceUsageStore,
- type UserWorkspace
+ workspaceUsageStore
} from '$lib/stores'
import { refreshExecutions } from '$lib/usage.svelte'
import { logFeatureUsage } from '$lib/utils/featureUsage'
import { scopedValue, tagged } from '$lib/utils/scopedValue'
- import { findWorkspaceAncestors } from '$lib/utils/workspaceHierarchy'
import { Button } from '$lib/components/common'
import Modal from '$lib/components/common/modal/Modal.svelte'
import { Tooltip } from '$lib/components/meltComponents'
@@ -30,51 +27,30 @@
let open = $state(false)
- // A fork's usage and tier resolve to its billing root while its member list is a
- // subset of the root's, so seats must come from the root or the cap is fork-sized
- // against root usage. `undefined` when the root isn't visible from here: the cap
- // is then unknowable, and the caller hides the meter rather than guessing.
- function billingRoot(workspace: string, all: UserWorkspace[]): string | undefined {
- const self = all.find((w) => w.id === workspace)
- if (!self) return undefined
- if (!self.parent_workspace_id) return workspace
- const top = findWorkspaceAncestors(workspace, all).at(-1)
- return top && !top.parent_workspace_id ? top.id : undefined
- }
+ // Seat count for a paid workspace, the basis of its included executions. The server
+ // resolves a fork to the workspace its plan is billed on and counts the seats there,
+ // because neither is answerable from here: a fork's member list is a subset of that
+ // root's, and a fork member need not be a member of the root at all.
+ const fetchSeats = tagged(
+ async (workspace: string) => (await WorkspaceService.getBillableSeats({ workspace })).seats
+ )
- // Seat count for a paid workspace, the basis of its included executions. Only the
- // user list is needed: `premium_info` carries the same usage number as
- // `workspaceUsageStore` but requires admin and only exists when Stripe is
- // configured, so it would leave regular members with no block at all.
- const fetchSeats = tagged(async (root: string) => {
- // Throws for a fork member with no seat in the root, which is the same answer as
- // an unresolvable root: leave the paid meter hidden.
- const users = await UserService.listUsers({ workspace: root })
- // Same basis as the backend's `count_paid_seats`: disabled members and service
- // accounts are not billed, so counting them inflates the cap and hides a real
- // overage. 1 developer = 1 seat, 2 operators = 1 seat.
- const billable = users.filter((u) => !u.disabled && !u.is_service_account)
- const developers = billable.filter((u) => !u.operator).length
- const operators = billable.length - developers
- return Math.ceil(developers + operators / 2)
- })
-
- const billingRootId = $derived.by(() => {
- const workspace = $workspaceStore
- if (!isCloudHosted() || !$isPremiumStore || !workspace) return undefined
- return billingRoot(workspace, $userWorkspaces ?? [])
- })
+ const meteredWorkspace = $derived(
+ isCloudHosted() && $isPremiumStore ? $workspaceStore : undefined
+ )
// The membership version is in the key so a change re-resolves the cap, but not in
// the tag: tagging by it would blank the bar on every change.
const seatsResource = resource(
() =>
- billingRootId ? { root: billingRootId, version: $workspaceMembershipVersion } : undefined,
- async (key) => (key ? await fetchSeats(key.root) : undefined)
+ meteredWorkspace
+ ? { workspace: meteredWorkspace, version: $workspaceMembershipVersion }
+ : undefined,
+ async (key) => (key ? await fetchSeats(key.workspace) : undefined)
)
const scopedSeats = scopedValue()
- const seats = $derived(scopedSeats(billingRootId, seatsResource.current))
+ const seats = $derived(scopedSeats(meteredWorkspace, seatsResource.current))
type QuotaKey = 'user' | 'workspace'
diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
index 782da1bb2c..8dd2937273 100644
--- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
+++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
@@ -41,7 +41,7 @@
import { sendUserToast } from '$lib/toast'
import { clone, emptyString, encodeState, hasUnsavedChanges } from '$lib/utils'
import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile'
- import { Slack, Target } from 'lucide-svelte'
+ import { ExternalLink, Slack, Target } from 'lucide-svelte'
import SidebarNavigation from '$lib/components/common/sidebar/SidebarNavigation.svelte'
import PremiumInfo from '$lib/components/settings/PremiumInfo.svelte'
@@ -1472,9 +1472,30 @@
This workspace is a fork of {currentWorkspace.parent_workspace_id}. It runs
on the parent's plan and its executions count toward the parent's usage and bill,
- so there is no separate subscription here. Manage billing, seats, and quotas from
- the parent workspace's settings.
+ so it is never invoiced separately. Manage billing, seats, and quotas from the
+ parent workspace's settings.
+ {#if plan}
+
+
+ It is on a paid plan that is billed on its own, so this workspace is paid for
+ twice. Cancel that subscription in the customer portal to keep only
+ {currentWorkspace.parent_workspace_id}'s plan. This workspace keeps
+ running either way, on the parent's plan.
+ {#if customer_id}
+
Older matches were truncated from this search, try refining your filters to get
more precise results.
From 419d3adb6c785a3fa8408f05a7f0b0f8d7f03920 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel
Date: Sat, 29 Aug 2026 11:16:48 +0200
Subject: [PATCH 012/310] chore: bump tantivy to 0.27 and pin argon2 to 0.5
(#10890)
* chore: bump tantivy to 0.27 and pin argon2 to 0.5
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_016ijGGPCFkYhVzisFexAHYx
* chore: pin tantivy to the merged fork main head
---------
Co-authored-by: Claude Opus 5 (1M context)
---
backend/Cargo.lock | 186 +++++++++++++--------------------------------
backend/Cargo.toml | 7 +-
2 files changed, 59 insertions(+), 134 deletions(-)
diff --git a/backend/Cargo.lock b/backend/Cargo.lock
index 028d42e9ce..7c8d40ad05 100644
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -23,7 +23,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
dependencies = [
- "crypto-common 0.1.7",
+ "crypto-common",
"generic-array",
]
@@ -261,13 +261,13 @@ dependencies = [
[[package]]
name = "argon2"
-version = "0.6.0"
+version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c"
+checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072"
dependencies = [
"base64ct",
- "blake2 0.11.0",
- "cpufeatures 0.3.1",
+ "blake2",
+ "cpufeatures 0.2.17",
"password-hash",
]
@@ -1824,15 +1824,6 @@ dependencies = [
"digest 0.10.7",
]
-[[package]]
-name = "blake2"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f"
-dependencies = [
- "digest 0.11.3",
-]
-
[[package]]
name = "blake3"
version = "1.8.7"
@@ -1865,15 +1856,6 @@ dependencies = [
"generic-array",
]
-[[package]]
-name = "block-buffer"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
-dependencies = [
- "hybrid-array",
-]
-
[[package]]
name = "block-modes"
version = "0.8.1"
@@ -2408,7 +2390,7 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
- "crypto-common 0.1.7",
+ "crypto-common",
"inout",
]
@@ -2472,12 +2454,6 @@ dependencies = [
"cc",
]
-[[package]]
-name = "cmov"
-version = "0.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
-
[[package]]
name = "cms"
version = "0.2.3"
@@ -2842,15 +2818,6 @@ dependencies = [
"typenum",
]
-[[package]]
-name = "crypto-common"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
-dependencies = [
- "hybrid-array",
-]
-
[[package]]
name = "csv"
version = "1.4.0"
@@ -2881,15 +2848,6 @@ dependencies = [
"cipher 0.4.4",
]
-[[package]]
-name = "ctutils"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e"
-dependencies = [
- "cmov",
-]
-
[[package]]
name = "curl-sys"
version = "0.4.90+curl-8.21.0"
@@ -3455,7 +3413,7 @@ dependencies = [
"arrow",
"arrow-buffer",
"base64 0.22.1",
- "blake2 0.10.6",
+ "blake2",
"blake3",
"chrono",
"datafusion-common",
@@ -3731,9 +3689,9 @@ dependencies = [
[[package]]
name = "datasketches"
-version = "0.2.0"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745"
+checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d"
[[package]]
name = "debug-helper"
@@ -4497,21 +4455,10 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"const-oid",
- "crypto-common 0.1.7",
+ "crypto-common",
"subtle",
]
-[[package]]
-name = "digest"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
-dependencies = [
- "block-buffer 0.12.1",
- "crypto-common 0.2.2",
- "ctutils",
-]
-
[[package]]
name = "dirs"
version = "4.0.0"
@@ -5175,6 +5122,12 @@ dependencies = [
"syn 2.0.119",
]
+[[package]]
+name = "frostem"
+version = "1.20260821.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08ed6437a2ed7fc408115e25cdb41e15ba4b17742a4c8d3d1b5276fe9b687209"
+
[[package]]
name = "fs3"
version = "0.5.0"
@@ -6186,15 +6139,6 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
-[[package]]
-name = "hybrid-array"
-version = "0.4.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b"
-dependencies = [
- "typenum",
-]
-
[[package]]
name = "hyper"
version = "0.14.32"
@@ -7379,15 +7323,6 @@ dependencies = [
"hashbrown 0.15.5",
]
-[[package]]
-name = "lru"
-version = "0.16.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39"
-dependencies = [
- "hashbrown 0.16.1",
-]
-
[[package]]
name = "lru"
version = "0.18.3"
@@ -7423,9 +7358,9 @@ dependencies = [
[[package]]
name = "lz4_flex"
-version = "0.13.1"
+version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
+checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226"
[[package]]
name = "lzma-sys"
@@ -7849,9 +7784,9 @@ dependencies = [
[[package]]
name = "murmurhash32"
-version = "0.3.1"
+version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
+checksum = "a8afc6df942f4c022d70c5725e18df1a705773870e5b7f96fde94ca0334ce77a"
[[package]]
name = "mysql-common-derive"
@@ -8842,7 +8777,7 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
[[package]]
name = "ownedbytes"
version = "0.9.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c"
+source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a"
dependencies = [
"stable_deref_trait",
]
@@ -8989,12 +8924,13 @@ dependencies = [
[[package]]
name = "password-hash"
-version = "0.6.1"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b"
+checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166"
dependencies = [
- "getrandom 0.4.3",
- "phc",
+ "base64ct",
+ "rand_core 0.6.4",
+ "subtle",
]
[[package]]
@@ -9138,17 +9074,6 @@ dependencies = [
"phf 0.11.3",
]
-[[package]]
-name = "phc"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892"
-dependencies = [
- "base64ct",
- "ctutils",
- "getrandom 0.4.3",
-]
-
[[package]]
name = "phf"
version = "0.11.3"
@@ -10660,16 +10585,6 @@ dependencies = [
"walkdir",
]
-[[package]]
-name = "rust-stemmers"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
-dependencies = [
- "serde",
- "serde_derive",
-]
-
[[package]]
name = "rust_decimal"
version = "1.42.1"
@@ -12803,12 +12718,12 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
name = "tantivy"
-version = "0.26.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c"
+version = "0.27.0"
+source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a"
dependencies = [
"aho-corasick",
"arc-swap",
- "base64 0.22.1",
+ "base64 0.23.1",
"bitpacking",
"bon",
"byteorder",
@@ -12819,20 +12734,20 @@ dependencies = [
"downcast-rs",
"fastdivide",
"fnv",
+ "frostem",
"fs4",
"htmlescape",
"itertools 0.14.0",
"levenshtein_automata",
"log",
- "lru 0.16.4",
- "lz4_flex 0.13.1",
+ "lru 0.18.3",
+ "lz4_flex 0.14.0",
"measure_time",
"memmap2",
"once_cell",
"oneshot",
"rayon",
"regex",
- "rust-stemmers",
"rustc-hash 2.1.3",
"serde",
"serde_json",
@@ -12849,22 +12764,23 @@ dependencies = [
"thiserror 2.0.20",
"time",
"typetag",
+ "unwrap-infallible",
"uuid",
"winapi",
]
[[package]]
name = "tantivy-bitpacker"
-version = "0.9.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c"
+version = "0.10.0"
+source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a"
dependencies = [
"bitpacking",
]
[[package]]
name = "tantivy-columnar"
-version = "0.6.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c"
+version = "0.7.0"
+source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a"
dependencies = [
"downcast-rs",
"fastdivide",
@@ -12878,8 +12794,8 @@ dependencies = [
[[package]]
name = "tantivy-common"
-version = "0.10.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c"
+version = "0.11.0"
+source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a"
dependencies = [
"async-trait",
"byteorder",
@@ -12901,8 +12817,8 @@ dependencies = [
[[package]]
name = "tantivy-query-grammar"
-version = "0.25.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c"
+version = "0.26.0"
+source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a"
dependencies = [
"fnv",
"nom",
@@ -12913,8 +12829,8 @@ dependencies = [
[[package]]
name = "tantivy-sstable"
-version = "0.6.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c"
+version = "0.7.0"
+source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a"
dependencies = [
"futures-util",
"itertools 0.14.0",
@@ -12926,8 +12842,8 @@ dependencies = [
[[package]]
name = "tantivy-stacker"
-version = "0.6.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c"
+version = "0.7.0"
+source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a"
dependencies = [
"murmurhash32",
"tantivy-common",
@@ -12935,8 +12851,8 @@ dependencies = [
[[package]]
name = "tantivy-tokenizer-api"
-version = "0.6.0"
-source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c"
+version = "0.7.0"
+source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a"
dependencies = [
"serde",
]
@@ -14229,7 +14145,7 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
dependencies = [
- "crypto-common 0.1.7",
+ "crypto-common",
"subtle",
]
@@ -14257,6 +14173,12 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+[[package]]
+name = "unwrap-infallible"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e497bb1f828cc9fb236722c2eaa100dcf201563f38f4da6252357a59037adf31"
+
[[package]]
name = "ureq"
version = "2.12.1"
diff --git a/backend/Cargo.toml b/backend/Cargo.toml
index a05bd109ae..3f614c9388 100644
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -477,7 +477,10 @@ rust-embed = { version = "^6", features = ["interpolate-folder-path"] }
mime_guess = "^2"
hex = "^0"
sql-builder = "^3"
-argon2 = "^0"
+# Pinned: `^0` floats to 0.6, which moved `password_hash::SaltString`, put
+# `rand_core` behind a feature and changed `hash_password`'s signature —
+# users_ee.rs is written against 0.5 and does not compile otherwise.
+argon2 = "0.5"
quick_cache = "^0"
rand = "=0.9.0"
rand_core = { version = "^0", features = ["std"] }
@@ -697,7 +700,7 @@ tikv-jemalloc-ctl = { version = "^0.5" }
triomphe = "^0"
pin-project-lite = "^0"
-tantivy = { git="https://github.com/windmill-labs/tantivy", rev="6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" }
+tantivy = { git="https://github.com/windmill-labs/tantivy", rev="ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" }
backon = "1.3.0"
From c8172480b0b1be6c57210212afc71d6ec8711235 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel
Date: Sat, 29 Aug 2026 11:53:21 +0200
Subject: [PATCH 013/310] fix: register every rotated service log file exactly
once (#10891)
* fix: register every rotated service log file exactly once
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_016QTxWg4Sx57UodA9RpFMJm
* chore: refresh sqlx cache for the log_file watermark query
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_016QTxWg4Sx57UodA9RpFMJm
* fix: skip service log files past the retention cutoff on catch-up
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_016QTxWg4Sx57UodA9RpFMJm
* refactor: name the shutdown flush for what it does and scope its doc claims
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_016QTxWg4Sx57UodA9RpFMJm
---------
Co-authored-by: Claude Opus 5 (1M context)
---
...01dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json | 22 +
...76dac75ba6df5f8eaa9f185300483e3ee36f.json} | 4 +-
backend/src/main.rs | 4 +-
backend/src/monitor.rs | 377 ++++++++++++------
4 files changed, 281 insertions(+), 126 deletions(-)
create mode 100644 backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json
rename backend/.sqlx/{query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json => query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json} (51%)
diff --git a/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json b/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json
new file mode 100644
index 0000000000..4481fa3dea
--- /dev/null
+++ b/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT max(log_ts) FROM log_file\n WHERE hostname = $1 AND log_ts < (SELECT max(log_ts) FROM log_file WHERE hostname = $1)",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "max",
+ "type_info": "Timestamp"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c"
+}
diff --git a/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json b/backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json
similarity index 51%
rename from backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json
rename to backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json
index 7df22ca7b7..976cdfec06 100644
--- a/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json
+++ b/backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7",
+ "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7",
"describe": {
"columns": [],
"parameters": {
@@ -17,5 +17,5 @@
},
"nullable": []
},
- "hash": "92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b"
+ "hash": "f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f"
}
diff --git a/backend/src/main.rs b/backend/src/main.rs
index 69f28e11ee..8803157d84 100644
--- a/backend/src/main.rs
+++ b/backend/src/main.rs
@@ -14,7 +14,7 @@ use monitor::{
reload_nuget_config_setting, reload_powershell_repo_pat_setting,
reload_powershell_repo_url_setting, reload_ruby_repos_setting,
reload_timeout_wait_result_setting, reload_workspace_registries_setting,
- send_current_log_file_to_object_store, send_logs_to_object_store, WORKERS_NAMES,
+ flush_pending_log_files_to_object_store, send_logs_to_object_store, WORKERS_NAMES,
};
use rand::Rng;
use sqlx::{Pool, Postgres};
@@ -1662,7 +1662,7 @@ Windmill Community Edition {GIT_VERSION}
} else {
tracing::info!("Nothing to do, exiting.");
}
- send_current_log_file_to_object_store(&conn, &hostname, &mode).await;
+ flush_pending_log_files_to_object_store(&conn, &hostname, &mode).await;
if let Some(db) = conn.as_sql() {
tracing::info!("Exiting connection pool");
diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs
index 6e0d50d30e..d640e1bda6 100644
--- a/backend/src/monitor.rs
+++ b/backend/src/monitor.rs
@@ -1220,32 +1220,60 @@ async fn sleep_until_next_minute_start_plus_one_s() {
}
use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE;
-async fn find_two_highest_files(hostname: &str) -> (Option, Option) {
+
+/// The minutely rolling appender names each file `.log.<%Y-%m-%d-%H-%M>`;
+/// anything else in the directory is not a rotated log file.
+fn parse_log_file_ts(file_name: &str) -> Option {
+ NaiveDateTime::parse_from_str(
+ file_name.rsplit('.').next()?,
+ windmill_common::tracing_init::LOG_TIMESTAMP_FMT,
+ )
+ .ok()
+}
+
+/// Oldest first. Readdir order is filesystem-dependent — tmpfs hands back the
+/// newest entry first, ext4 hashes the names — so the listing has to be sorted
+/// before anything picks a file out of it.
+fn sorted_log_files(file_names: impl Iterator) -> Vec<(NaiveDateTime, String)> {
+ let mut files = file_names
+ .filter_map(|name| parse_log_file_ts(&name).map(|ts| (ts, name)))
+ .collect::>();
+ files.sort();
+ files
+}
+
+/// Every log file but the newest one: that one is still being appended to, every
+/// older one is final.
+fn rotated_log_files(file_names: impl Iterator) -> Vec<(NaiveDateTime, String)> {
+ let mut files = sorted_log_files(file_names);
+ files.pop();
+ files
+}
+
+async fn read_log_file_names(hostname: &str) -> Vec {
let log_dir = format!("{}/{}/", *TMP_WINDMILL_LOGS_SERVICE, hostname);
- let rd_dir = tokio::fs::read_dir(log_dir).await;
- if let Ok(mut log_files) = rd_dir {
- let mut highest_file: Option = None;
- let mut second_highest_file: Option = None;
- while let Ok(Some(file)) = log_files.next_entry().await {
- let file_name = file
- .file_name()
- .to_str()
- .map(|x| x.to_string())
- .unwrap_or_default();
- if file_name > highest_file.clone().unwrap_or_default() {
- second_highest_file = highest_file;
- highest_file = Some(file_name);
- }
+ let mut rd_dir = match tokio::fs::read_dir(&log_dir).await {
+ Ok(rd_dir) => rd_dir,
+ Err(e) => {
+ tracing::error!("Error reading log files: {}, {:#?}", log_dir, e);
+ return vec![];
+ }
+ };
+ let mut file_names = vec![];
+ while let Ok(Some(file)) = rd_dir.next_entry().await {
+ if let Some(file_name) = file.file_name().to_str() {
+ file_names.push(file_name.to_string());
}
- (highest_file, second_highest_file)
- } else {
- tracing::error!(
- "Error reading log files: {}, {:#?}",
- *TMP_WINDMILL_LOGS_SERVICE,
- rd_dir.unwrap_err()
- );
- (None, None)
}
+ file_names
+}
+
+async fn list_log_files(hostname: &str) -> Vec<(NaiveDateTime, String)> {
+ sorted_log_files(read_log_file_names(hostname).await.into_iter())
+}
+
+async fn list_rotated_log_files(hostname: &str) -> Vec<(NaiveDateTime, String)> {
+ rotated_log_files(read_log_file_names(hostname).await.into_iter())
}
fn get_worker_group(mode: &Mode) -> Option {
@@ -1265,133 +1293,187 @@ pub fn send_logs_to_object_store(conn: &Connection, hostname: &str, mode: &Mode)
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(10));
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+ init_last_log_file_sent(&conn, &hostname).await;
sleep_until_next_minute_start_plus_one_s().await;
loop {
interval.tick().await;
- let (_, snd_highest_file) = find_two_highest_files(&hostname).await;
- send_log_file_to_object_store(
- &hostname,
- &mode,
- &worker_group,
- &conn,
- snd_highest_file,
- false,
- )
- .await;
+ let files = list_rotated_log_files(&hostname).await;
+ send_log_files_to_object_store(&hostname, &mode, &worker_group, &conn, files).await;
}
});
}
-pub async fn send_current_log_file_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) {
- tracing::info!("Sending current log file to object store");
- let (highest_file, _) = find_two_highest_files(hostname).await;
+pub async fn flush_pending_log_files_to_object_store(
+ conn: &Connection,
+ hostname: &str,
+ mode: &Mode,
+) {
+ tracing::info!("Sending pending log files to object store");
let worker_group = get_worker_group(&mode);
- send_log_file_to_object_store(hostname, mode, &worker_group, conn, highest_file, true).await;
-}
-
-fn get_now_and_str() -> (NaiveDateTime, String) {
- let ts = Utc::now().naive_utc();
- (
- ts,
- ts.format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT)
- .to_string(),
- )
+ // Nothing rotates after this, so the file still being appended to is registered
+ // here, along with any rotated one the loop had not reached yet. Bounded like the
+ // pool close that follows: a backlog against a slow object store would otherwise
+ // hold the process past its termination grace period. Whatever is left over is
+ // registered by the next run's catch-up.
+ let flush = async {
+ let files = list_log_files(hostname).await;
+ send_log_files_to_object_store(hostname, mode, &worker_group, conn, files).await;
+ };
+ if timeout(Duration::from_secs(15), flush).await.is_err() {
+ tracing::warn!("Could not send all pending log files in time (15s). Exiting anyway.");
+ }
}
lazy_static::lazy_static! {
static ref LAST_LOG_FILE_SENT: Arc>> = Arc::new(Mutex::new(None));
+ /// Serializes the periodic uploader against the shutdown flush. The uploader is a
+ /// detached task that keeps ticking while the flush runs and both walk the same
+ /// files, so without this both can clear the watermark for one file and count its
+ /// lines twice through the additive upsert.
+ static ref SENDING_LOG_FILES: tokio::sync::Mutex<()> = tokio::sync::Mutex::new(());
}
+fn last_log_file_sent() -> Option {
+ LAST_LOG_FILE_SENT.lock().ok().and_then(|ts| *ts)
+}
+
+/// Resume from what this host already registered, so a previous run's leftovers reach
+/// the object store rather than being dropped. Their line counts come out zero, this
+/// run having counted none of them, which only flattens their bars in the UI.
+///
+/// The newest registered minute is left out on purpose: the shutdown flush registers
+/// the file that was still open and the appender reopens that minute in append mode,
+/// so a restart inside it would otherwise strand everything written afterwards.
+///
+/// A row rewritten this way restores the object and sums the counters, but whether the
+/// indexers read it again depends on their single `log_ts >` cursor, which is not
+/// per-hostname: a minute at or below it stays out of search until it is re-indexed.
+async fn init_last_log_file_sent(conn: &Connection, hostname: &str) {
+ let Some(db) = conn.as_sql() else {
+ return;
+ };
+ match sqlx::query_scalar!(
+ "SELECT max(log_ts) FROM log_file
+ WHERE hostname = $1 AND log_ts < (SELECT max(log_ts) FROM log_file WHERE hostname = $1)",
+ hostname
+ )
+ .fetch_one(db)
+ .await
+ {
+ Ok(Some(ts)) => {
+ if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| {
+ last_log_file_sent.replace(ts);
+ }) {
+ tracing::error!("Error initializing last log file sent: {:?}", e);
+ }
+ }
+ Ok(None) => {}
+ Err(e) => tracing::error!("Error loading last log file sent: {:?}", e),
+ }
+}
+
+async fn send_log_files_to_object_store(
+ hostname: &str,
+ mode: &Mode,
+ worker_group: &Option,
+ conn: &Connection,
+ files: Vec<(NaiveDateTime, String)>,
+) {
+ let _guard = SENDING_LOG_FILES.lock().await;
+ let retention_cutoff =
+ Utc::now().naive_utc() - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS);
+ for (ts, file_name) in files {
+ if last_log_file_sent().is_some_and(|last| last >= ts) {
+ continue;
+ }
+ // A run coming back from a long outage still finds its predecessor's files on
+ // disk. Registering one past the retention cutoff inserts a row
+ // `delete_expired_items` drops on its next pass, once the indexers have already
+ // paid to parse it.
+ if ts < retention_cutoff {
+ continue;
+ }
+ // Stop at the first failure rather than moving on: both indexers walk
+ // `log_file` with a `log_ts > watermark` cursor, so a row that lands after
+ // a newer one is never picked up.
+ if !send_log_file_to_object_store(hostname, mode, worker_group, conn, &file_name, ts).await
+ {
+ break;
+ }
+ }
+}
+
+/// Returns whether the file ended up registered in `log_file`.
async fn send_log_file_to_object_store(
hostname: &str,
mode: &Mode,
worker_group: &Option,
conn: &Connection,
- snd_highest_file: Option,
- use_now: bool,
-) {
- if let Some(highest_file) = snd_highest_file {
- //parse datetime frome file xxxx.yyyy-MM-dd-HH-mm
- let (ts, ts_str) = if use_now {
- get_now_and_str()
- } else {
- highest_file
- .split(".")
- .last()
- .and_then(|x| {
- NaiveDateTime::parse_from_str(
- x,
- windmill_common::tracing_init::LOG_TIMESTAMP_FMT,
- )
- .ok()
- .map(|y| (y, x.to_string()))
- })
- .unwrap_or_else(get_now_and_str)
- };
+ file_name: &str,
+ ts: NaiveDateTime,
+) -> bool {
+ #[cfg(feature = "parquet")]
+ if let Some(s3_client) = windmill_object_store::get_object_store().await {
+ let path = std::path::Path::new(&*TMP_WINDMILL_LOGS_SERVICE)
+ .join(hostname)
+ .join(file_name);
- let exists = LAST_LOG_FILE_SENT.lock().map(|last_log_file_sent| {
- last_log_file_sent
- .map(|last_log_file_sent| last_log_file_sent >= ts)
- .unwrap_or(false)
- });
-
- if exists.unwrap_or(false) {
- return;
- }
-
- #[cfg(feature = "parquet")]
- let s3_client = windmill_object_store::get_object_store().await;
- #[cfg(feature = "parquet")]
- if let Some(s3_client) = s3_client {
- let path = std::path::Path::new(&*TMP_WINDMILL_LOGS_SERVICE)
- .join(hostname)
- .join(&highest_file);
-
- //read file as byte stream
- let bytes = tokio::fs::read(&path).await;
- if let Err(e) = bytes {
+ //read file as byte stream
+ let bytes = match tokio::fs::read(&path).await {
+ Ok(bytes) => bytes,
+ Err(e) => {
tracing::error!("Error reading log file: {:?}", e);
- return;
+ return false;
}
- let path = windmill_object_store::object_store_reexports::Path::from_url_path(format!(
- "{}{hostname}/{highest_file}",
- windmill_common::tracing_init::LOGS_SERVICE
- ));
- if let Err(e) = path {
+ };
+ let path = windmill_object_store::object_store_reexports::Path::from_url_path(format!(
+ "{}{hostname}/{file_name}",
+ windmill_common::tracing_init::LOGS_SERVICE
+ ));
+ let path = match path {
+ Ok(path) => path,
+ Err(e) => {
tracing::error!("Error creating log file path: {:?}", e);
- return;
- }
- if let Err(e) = s3_client.put(&path.unwrap(), bytes.unwrap().into()).await {
- tracing::error!("Error sending logs to object store: {:?}", e);
+ return false;
}
+ };
+ if let Err(e) = s3_client.put(&path, bytes.into()).await {
+ tracing::error!("Error sending logs to object store: {:?}", e);
+ return false;
}
+ }
- let (ok_lines, err_lines) = read_log_counters(ts_str);
+ let ts_str = ts
+ .format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT)
+ .to_string();
+ let (ok_lines, err_lines) = read_log_counters(ts_str);
- if let Some(db) = conn.as_sql() {
- match timeout(Duration::from_secs(10), sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)
- VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)
- ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7",
- hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, true)
- .execute(db)).await {
- Ok(Ok(_)) => {
- if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| {
- last_log_file_sent.replace(ts);
- }) {
- tracing::error!("Error updating last log file sent: {:?}", e);
- }
- tracing::info!("Log file sent: {}", highest_file);
- }
- Ok(Err(e)) => {
- tracing::error!("Error inserting log file: {:?}", e);
- }
- Err(e) => {
- tracing::error!("Error inserting log file, timeout elapsed: {:?}", e);
- }
+ let Some(db) = conn.as_sql() else {
+ // not sending log file to object store in agent mode
+ return false;
+ };
+
+ match timeout(Duration::from_secs(10), sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)
+ VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)
+ ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7",
+ hostname, mode.to_string(), worker_group.clone(), ts, file_name, ok_lines as i64, err_lines as i64, true)
+ .execute(db)).await {
+ Ok(Ok(_)) => {
+ if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| {
+ last_log_file_sent.replace(ts);
+ }) {
+ tracing::error!("Error updating last log file sent: {:?}", e);
}
- } else {
- // tracing::warn!("Not sending log file to object store in agent mode");
- ()
+ tracing::info!("Log file sent: {}", file_name);
+ true
+ }
+ Ok(Err(e)) => {
+ tracing::error!("Error inserting log file: {:?}", e);
+ false
+ }
+ Err(e) => {
+ tracing::error!("Error inserting log file, timeout elapsed: {:?}", e);
+ false
}
}
}
@@ -6831,3 +6913,54 @@ mod zombie_worker_memory_pct_tests {
);
}
}
+
+#[cfg(test)]
+mod log_file_listing_tests {
+ use super::{rotated_log_files, sorted_log_files};
+
+ fn names(files: Vec<(chrono::NaiveDateTime, String)>) -> Vec {
+ files.into_iter().map(|(_, n)| n).collect()
+ }
+
+ /// A directory read newest-entry-first is what tmpfs actually hands back.
+ #[test]
+ fn orders_by_minute_whatever_order_readdir_used() {
+ let newest_first = [
+ "h.log.2026-08-29-06-49",
+ "h.log.2026-08-29-06-46",
+ "h.log.2026-08-29-06-48",
+ "h.log.2026-08-29-06-47",
+ ];
+ assert_eq!(
+ names(sorted_log_files(newest_first.iter().map(|x| x.to_string()))),
+ vec![
+ "h.log.2026-08-29-06-46",
+ "h.log.2026-08-29-06-47",
+ "h.log.2026-08-29-06-48",
+ "h.log.2026-08-29-06-49",
+ ]
+ );
+ assert_eq!(
+ names(rotated_log_files(
+ newest_first.iter().map(|x| x.to_string())
+ )),
+ vec![
+ "h.log.2026-08-29-06-46",
+ "h.log.2026-08-29-06-47",
+ "h.log.2026-08-29-06-48",
+ ]
+ );
+ }
+
+ #[test]
+ fn drops_names_that_are_not_rotated_log_files() {
+ let files = sorted_log_files(
+ ["h.log", "not-a-log-file", "h.log.2026-08-29-06-46"]
+ .iter()
+ .map(|x| x.to_string()),
+ );
+ assert_eq!(files.len(), 1);
+ assert_eq!(files[0].1, "h.log.2026-08-29-06-46");
+ assert_eq!(files[0].0.to_string(), "2026-08-29 06:46:00");
+ }
+}
From 338d75cc5227e352cb84828c99bfd3b984cf0fa5 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel
Date: Sat, 29 Aug 2026 18:47:53 +0200
Subject: [PATCH 014/310] feat: serve service log context from parquet and
retire the raw log files (#10892)
* feat: serve service log context from the parquet store and retire the raw files
* fix: keep the log ingest cursor in the store and stream file rebuilds
* fix: roll back a partial index rebuild and move the cursor before the commit
* fix: make the index rebuild idempotent and repair a cursor the index never caught up with
* fix: seed the indexed cursor on upgrade and after a rebuild
* fix: fail the indexing pass on an unreadable cursor instead of reading it as absent
* docs: record what keeps both known_ts entries, not the path main removed
* chore: update ee-repo-ref to 466eb1830879052a5d042295256a78375bee916d
This commit updates the EE repository reference after PR #754 was merged in windmill-ee-private.
Previous ee-repo-ref: ddb3a536b8d85c134c01f87da7783baaa204a6d1
New ee-repo-ref: 466eb1830879052a5d042295256a78375bee916d
Automated by sync-ee-ref workflow.
---------
Co-authored-by: windmill-internal-app[bot]
---
...34f1befe6b16eee363c843faa1f836d53ca8d.json | 29 +++++
backend/ee-repo-ref.txt | 2 +-
backend/windmill-api/src/service_logs.rs | 104 +++++++++++++++---
3 files changed, 116 insertions(+), 19 deletions(-)
create mode 100644 backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json
diff --git a/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json b/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json
new file mode 100644
index 0000000000..d64f007d7b
--- /dev/null
+++ b/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json
@@ -0,0 +1,29 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT mode::text AS \"mode!\", log_ts FROM log_file WHERE hostname = $1 AND file_path = $2 ORDER BY log_ts DESC LIMIT 1",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "mode!",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "log_ts",
+ "type_info": "Timestamp"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ null,
+ false
+ ]
+ },
+ "hash": "daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d"
+}
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index 5c23919dc8..cce83bbdff 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-ad9e899dfd2ee4e3d18ecf06d016f821968c5a83
\ No newline at end of file
+466eb1830879052a5d042295256a78375bee916d
diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs
index 99dc7c41a8..a54f275f46 100644
--- a/backend/windmill-api/src/service_logs.rs
+++ b/backend/windmill-api/src/service_logs.rs
@@ -88,6 +88,66 @@ async fn list_files(
Ok(Json(rows))
}
+/// Rebuild one source log file from the columnar store.
+///
+/// Not the original bytes: the store holds a line's fields rather than its text,
+/// so the JSON is re-serialized here and key order and whitespace are this
+/// writer's. Everything a reader can see survives — the drawer this feeds
+/// renders a prettified view of each line either way, and a line that was never
+/// JSON comes back exactly as it was written.
+#[cfg(all(feature = "tantivy", feature = "private"))]
+async fn get_log_file_from_store(
+ db: &DB,
+ store: &windmill_indexer::service_logs_store_ee::Store,
+ path: &str,
+) -> windmill_common::error::Result {
+ let (hostname, file_name) = path
+ .split_once('/')
+ .ok_or_else(|| Error::BadRequest("Invalid path".to_string()))?;
+
+ // The store is partitioned by day and mode, neither of which the path
+ // carries. `log_file` names both, and its primary key starts with hostname.
+ let file = sqlx::query!(
+ // `mode!` because the column is NOT NULL and only the cast makes sqlx
+ // think otherwise; a silent default would look up a `mode=` partition
+ // that matches nothing and read as a missing file.
+ "SELECT mode::text AS \"mode!\", log_ts FROM log_file WHERE hostname = $1 AND file_path = $2 ORDER BY log_ts DESC LIMIT 1",
+ hostname,
+ file_name
+ )
+ .fetch_optional(db)
+ .await?
+ .ok_or_else(|| Error::NotFound(format!("File {path} not found")))?;
+
+ // A row registered by this version carries the minute in the file's own name,
+ // so the two agree and the second is redundant. One written before the
+ // uploader derived `log_ts` from the name carries a wall clock instead, and
+ // those outlive an upgrade by the retention period — which is also what makes
+ // the `ORDER BY` above worth having. The name is authoritative, so both go.
+ let mut known_ts = vec![chrono::DateTime::from_naive_utc_and_offset(
+ file.log_ts,
+ chrono::Utc,
+ )];
+ if let Some(named) = file_name.rsplit('.').next().and_then(|s| {
+ chrono::NaiveDateTime::parse_from_str(s, windmill_common::tracing_init::LOG_TIMESTAMP_FMT)
+ .ok()
+ }) {
+ known_ts.push(chrono::DateTime::from_naive_utc_and_offset(
+ named,
+ chrono::Utc,
+ ));
+ }
+
+ let text = windmill_indexer::service_logs_store_ee::read_log_file(
+ store, &file.mode, hostname, file_name, &known_ts,
+ )
+ .await
+ .map_err(|e| Error::internal_err(format!("Error reading the service log store: {e}")))?
+ .ok_or_else(|| Error::NotFound(format!("File {path} not found")))?;
+
+ Ok(content_plain(Body::from(text)))
+}
+
async fn get_log_file(
authed: ApiAuthed,
Extension(db): Extension,
@@ -104,27 +164,30 @@ async fn get_log_file(
let s3_client = windmill_object_store::get_object_store().await;
#[cfg(feature = "parquet")]
if let Some(s3_client) = s3_client {
- let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path);
- let file = s3_client
+ use windmill_object_store::object_store_reexports::ObjectStoreError;
+
+ // The raw file, for as long as it is there. It outlives its ingestion by
+ // one indexer pass at most, so this covers the most recent minutes of a
+ // host's logs byte for byte; everything older is rebuilt from the store.
+ let object_path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path);
+ match s3_client
.get(&windmill_object_store::object_store_reexports::Path::from(
- path,
+ object_path,
))
- .await;
- match file {
- Ok(file) => {
- let bytes = file.bytes().await;
- match bytes {
- Ok(bytes) => {
- return Ok(content_plain(Body::from(bytes::Bytes::from(bytes))));
- }
- Err(e) => {
- return Err(Error::internal_err(format!(
- "Error pulling the bytes: {}",
- e
- )));
- }
+ .await
+ {
+ Ok(file) => match file.bytes().await {
+ Ok(bytes) => {
+ return Ok(content_plain(Body::from(bytes::Bytes::from(bytes))));
}
- }
+ Err(e) => {
+ return Err(Error::internal_err(format!(
+ "Error pulling the bytes: {}",
+ e
+ )));
+ }
+ },
+ Err(ObjectStoreError::NotFound { .. }) => {}
Err(e) => {
return Err(Error::internal_err(format!(
"Error fetching the file: {}",
@@ -132,6 +195,11 @@ async fn get_log_file(
)));
}
}
+
+ #[cfg(all(feature = "tantivy", feature = "private"))]
+ return get_log_file_from_store(&db, &s3_client, &path).await;
+ #[cfg(not(all(feature = "tantivy", feature = "private")))]
+ return Err(Error::NotFound(format!("File {path} not found")));
}
let full_path = format!("{}{}", *TMP_WINDMILL_LOGS_SERVICE, path);
// SECURITY (defense in depth): refuse to read through a symlink so a planted
From 815de49e2322f85ca92b1e41a2bcd22591ebe93f Mon Sep 17 00:00:00 2001
From: Ruben Fiszel
Date: Sat, 29 Aug 2026 19:39:14 +0200
Subject: [PATCH 015/310] feat: make the service log retention period an
instance setting (#10889)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat: make the service log retention period an instance setting
Service log retention was a hardcoded 14 days with no override, unlike job retention. It
becomes the `service_log_retention_secs` global setting (env `SERVICE_LOG_RETENTION_SECS`,
default unchanged at 14 days), reloaded on change like the other retention settings.
The constant becomes `DEFAULT_SERVICE_LOG_RETENTION_SECS` and every reader goes through
`service_log_retention_secs()`, so the `log_file` sweep, the object-storage orphan scan, the
columnar store's compaction and pruning, the retrieval clamp and the search index's trim
window all follow the configured value.
Loaded outside `initial_load`'s `server_mode` guard: a dedicated indexer trims the search
index to a window derived from this value and is not a server.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN
* fix: never let a non-positive service log retention expire every log
Every service log cutoff is `now - retention`, so a `0` or negative window puts the cutoff
at or after `now` and the next sweep reads the whole history as expired — deleting the
`log_file` rows and their object-storage files irreversibly.
`0` is reachable two ways now that the window is configurable: it is what an operator types
by analogy with the job retention period sitting directly above it, where `0` does mean keep
forever; and `SecondsInput` writes a `0` into a field that was merely focused, so saving the
Jobs panel is enough. Service logs always have a window, so clamp an unusable value back to
the default in the accessor every reader already goes through. The upper bound is where
`chrono::Duration::seconds` panics, which would abort the sweep that reads it.
The settings field rejects a non-positive value rather than silently correcting it, and its
description now names the database rows too — they are swept on every instance, including
one with no object storage configured.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN
* fix: address review findings on the service log retention setting
- Bound the monitor's `log_file` sweep. Every process rotates a log file a minute, so lowering
the retention can make one ordinary setting change expire millions of rows; the unbounded
`DELETE ... RETURNING` materialized all of them, and their deletion futures, in a single
tick. Batched like the settings-page cleanup on the same table.
- Make the retention atomic private and give it one writer, so a value that would expire every
service log cannot reach a cutoff by any path, and say so in the log when one is rejected
rather than falling back silently.
- Cap the retention at a century. The previous ceiling only bounded `TimeDelta` construction,
while consumers compute `now - retention`, which panics past year 262143, and build a
Postgres interval that overflows well before the old cap.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN
* fix: cap an oversized service log retention instead of shortening it
The two unusable directions were landing on the same fallback, so configuring a retention
above the ceiling silently produced 14 days — deleting logs the operator had asked to keep
for longer. Too large now caps at the maximum, which preserves that intent; only a
non-positive value, which would expire everything and has no upward reading, falls back to
the default.
Also bound the `log_file` drain to ten batches per pass: `monitor_db` runs under a 600s
timeout that cancels every maintenance future in the same `join!` and reports a critical
error, so a backlog large enough to need batching has to drain across ticks, the way the
neighbouring sweeps already do. The settings field carries the upper bound too, and the
superseded query's offline entry is dropped.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN
* fix: route the new log-file registration cutoff through the retention accessor
`send_log_files_to_object_store` arrived on main while this branch was open and reads the
retention directly. The atomic behind it is private now, so it goes through the accessor like
every other consumer — which also means the cutoff it uses to skip registering already-expired
files follows the configured retention rather than a fixed two weeks.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN
* fix: say why every mode loads the service log retention setting
A worker registers its rotated log files against the retention cutoff, so the comment naming
only the indexer no longer covers why the setting sits outside the `server_mode` guard.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN
* fix: file service log retention under Monitoring, not Jobs
Service logs are the Windmill processes' own logs — every process rotates and registers its
own, no job involved — so the Jobs panel was grouping by the shape of the widget rather than
by the subject. It sits under Monitoring now, beside the Indexer panel that holds the other
service-log window.
Its own section rather than inside that panel: the panel is badged EE, while this governs the
database sweep that runs on every instance.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN
* chore: update ee-repo-ref to a6e3533b26195918a17fea58646f71d2bbcde288
This commit updates the EE repository reference after PR #752 was merged in windmill-ee-private.
Previous ee-repo-ref: 1d93da24bd166b9a5a5cc204034a1d35ffc88474
New ee-repo-ref: a6e3533b26195918a17fea58646f71d2bbcde288
Automated by sync-ee-ref workflow.
* feat: say on the service logs page where the logs actually are
The retention number alone does not tell an operator what it governs, and the answer differs
by instance. Two states are worth calling out because they are the ones where retention does
not mean what it looks like:
Without instance object storage, each process keeps its files on its own disk. The page lists
what every host wrote, since the rows are in the shared database, but can only open the files
of the replica serving the request, and a host's files go with it when it is replaced.
With object storage but "Delete logs from s3 periodically" off — the backend default, since
uploads are gated on a store existing while deletions are gated on that toggle — expiring a
log removes the row and the local file and leaves the uploaded copy behind for good.
The retention field itself now names every copy it covers and says that full-text search
reaches back at most that far, and less when the indexer's own window is shorter.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN
* fix: describe raw log files as the transient copy they became
Retiring the raw files landed while this was being written: the indexer now deletes each one
as soon as it is ingested, and the log viewer rebuilds a file from the columnar store once the
raw copy is gone. So the durable copy is the store, and warning that an uploaded file is kept
forever when periodic s3 deletion is off only holds where no indexer runs to ingest it.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN
* chore: point ee-repo-ref at the EE compile fix
EE main does not build on its own: extracting the index-window expression and adding a fourth
copy of it landed in separate PRs that never conflicted textually. windmill-ee-private#756 is
the one-line fix; this pins it so CI has a tree that compiles.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN
---------
Co-authored-by: Claude Opus 5 (1M context)
Co-authored-by: windmill-internal-app[bot]
---
...90c4b8506acefc85ef5b2f92a7bc451b1c5e.json} | 5 +-
backend/ee-repo-ref.txt | 2 +-
backend/src/main.rs | 11 +-
backend/src/monitor.rs | 106 ++++++++++++++----
.../windmill-api-settings/src/log_cleanup.rs | 13 ++-
.../windmill-common/src/global_settings.rs | 1 +
backend/windmill-common/src/indexer.rs | 72 ++++++++++++
backend/windmill-common/src/lib.rs | 50 ++++++++-
.../lib/components/InstanceSettings.svelte | 26 +++++
.../src/lib/components/instanceSettings.ts | 25 +++++
10 files changed, 275 insertions(+), 36 deletions(-)
rename backend/.sqlx/{query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json => query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json} (51%)
diff --git a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json
similarity index 51%
rename from backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json
rename to backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json
index 24a5ee62dd..42bac71afd 100644
--- a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json
+++ b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname",
+ "query": "DELETE FROM log_file WHERE (hostname, log_ts) IN (\n SELECT hostname, log_ts FROM log_file\n WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval\n LIMIT $2\n ) RETURNING file_path, hostname",
"describe": {
"columns": [
{
@@ -16,6 +16,7 @@
],
"parameters": {
"Left": [
+ "Int8",
"Int8"
]
},
@@ -24,5 +25,5 @@
false
]
},
- "hash": "94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033"
+ "hash": "0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e"
}
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index cce83bbdff..6f56f0ab9a 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-466eb1830879052a5d042295256a78375bee916d
+e6483ff5a289521405912d95be5c5ba064bedb38
diff --git a/backend/src/main.rs b/backend/src/main.rs
index 8803157d84..c782e1ad97 100644
--- a/backend/src/main.rs
+++ b/backend/src/main.rs
@@ -61,8 +61,9 @@ use windmill_common::{
SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING,
SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING,
SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING,
- SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
- UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING,
+ SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING,
+ TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING,
+ UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING,
WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING,
WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING,
WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING,
@@ -143,7 +144,8 @@ use crate::monitor::{
reload_pip_index_url_setting, reload_retention_period_setting,
reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting,
reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting,
- reload_sandbox_registry_auth_setting, reload_scim_token_setting, reload_smtp_config,
+ reload_sandbox_registry_auth_setting, reload_scim_token_setting,
+ reload_service_log_retention_secs_setting, reload_smtp_config,
reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting,
reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting,
reload_worker_config, MonitorIteration,
@@ -1953,6 +1955,9 @@ async fn process_notify_event(
}
TIMEOUT_WAIT_RESULT_SETTING => reload_timeout_wait_result_setting(conn).await,
RETENTION_PERIOD_SECS_SETTING => reload_retention_period_setting(conn).await,
+ SERVICE_LOG_RETENTION_SECS_SETTING => {
+ reload_service_log_retention_secs_setting(conn).await
+ }
RETENTION_PERIOD_SECS_OVERRIDES_SETTING => {
if let Err(e) = load_retention_period_overrides(db).await {
tracing::error!("Error loading per-workspace retention overrides: {e:#}");
diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs
index d640e1bda6..c0748a3798 100644
--- a/backend/src/monitor.rs
+++ b/backend/src/monitor.rs
@@ -70,11 +70,11 @@ use windmill_common::{
RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING,
SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING,
SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING,
- SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
- UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING,
- WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING,
- WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING,
- WORKSPACE_MAX_QUEUED_JOBS_SETTING,
+ SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING,
+ TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING,
+ UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING,
+ WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING,
+ WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING,
},
indexer::load_indexer_config,
jobs::delete_jobs,
@@ -97,10 +97,10 @@ use windmill_common::{
KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE,
CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED,
CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
- HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES,
- JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
- MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
- SERVICE_LOG_RETENTION_SECS, STORE_AUDIT_LOGS_S3,
+ DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, JOB_RETENTION_SECS,
+ JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED,
+ METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED,
+ OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3,
};
use windmill_common::{
client::AuthedClient,
@@ -475,6 +475,19 @@ pub async fn initial_load(
|v: Option| async move { HUB_API_SECRET.store(std::sync::Arc::new(v)) },
);
+ // Outside the `server_mode` guard below: every mode reads this. A worker registers its
+ // rotated log files against the cutoff, and a dedicated indexer trims the search index to a
+ // window derived from it — neither is a server.
+ pass.setting(SERVICE_LOG_RETENTION_SECS_SETTING, true, |v| async move {
+ windmill_common::set_service_log_retention_secs(parse_setting_value::(
+ v,
+ SERVICE_LOG_RETENTION_SECS_SETTING,
+ "SERVICE_LOG_RETENTION_SECS",
+ DEFAULT_SERVICE_LOG_RETENTION_SECS,
+ |x| x,
+ ))
+ });
+
if server_mode {
pass.setting(RETENTION_PERIOD_SECS_SETTING, true, |v| async move {
JOB_RETENTION_SECS.store(
@@ -1380,8 +1393,8 @@ async fn send_log_files_to_object_store(
files: Vec<(NaiveDateTime, String)>,
) {
let _guard = SENDING_LOG_FILES.lock().await;
- let retention_cutoff =
- Utc::now().naive_utc() - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS);
+ let retention_cutoff = Utc::now().naive_utc()
+ - chrono::Duration::seconds(windmill_common::service_log_retention_secs());
for (ts, file_name) in files {
if last_log_file_sent().is_some_and(|last| last >= ts) {
continue;
@@ -1661,6 +1674,13 @@ pub async fn trim_resource_versions(db: &DB) -> () {
}
}
+/// Matches the batch the settings-page cleanup uses for the same table.
+const SERVICE_LOG_DELETE_BATCH: i64 = 2_000;
+/// Batches per pass. `monitor_db` runs under a 600s timeout that cancels every maintenance
+/// future in the same `join!` and reports a critical error, so a large backlog has to drain
+/// across ticks rather than inside one, the way the neighbouring sweeps already do.
+const SERVICE_LOG_DELETE_MAX_BATCHES: usize = 10;
+
pub async fn delete_expired_items(db: &DB) -> () {
let expired_tokens_r = sqlx::query_as!(
TokenRow,
@@ -1743,23 +1763,48 @@ pub async fn delete_expired_items(db: &DB) -> () {
Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
}
- match sqlx::query_as!(
- LogFile,
- "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname",
- SERVICE_LOG_RETENTION_SECS,
- )
- .fetch_all(db)
- .await
- {
- Ok(log_files_to_delete) => {
+ // Batched: every process rotates a log file a minute, so lowering the retention makes one
+ // ordinary setting change expire millions of rows at once. An unbounded `DELETE ...
+ // RETURNING` would materialize all of them, and their deletion futures, in this one tick.
+ for _ in 0..SERVICE_LOG_DELETE_MAX_BATCHES {
+ let batch = sqlx::query_as!(
+ LogFile,
+ "DELETE FROM log_file WHERE (hostname, log_ts) IN (
+ SELECT hostname, log_ts FROM log_file
+ WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval
+ LIMIT $2
+ ) RETURNING file_path, hostname",
+ windmill_common::service_log_retention_secs(),
+ SERVICE_LOG_DELETE_BATCH,
+ )
+ .fetch_all(db)
+ .await;
+
+ match batch {
+ Ok(log_files_to_delete) => {
+ if log_files_to_delete.is_empty() {
+ break;
+ }
+ let n = log_files_to_delete.len();
let paths = log_files_to_delete
.iter()
.map(|f| format!("{}/{}", f.hostname, f.file_path))
.collect();
- delete_log_files_from_disk_and_store(paths, &*TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await;
-
+ delete_log_files_from_disk_and_store(
+ paths,
+ &*TMP_WINDMILL_LOGS_SERVICE,
+ windmill_common::tracing_init::LOGS_SERVICE,
+ )
+ .await;
+ if (n as i64) < SERVICE_LOG_DELETE_BATCH {
+ break;
+ }
+ }
+ Err(e) => {
+ tracing::error!("Error deleting log file: {:?}", e);
+ break;
+ }
}
- Err(e) => tracing::error!("Error deleting log file: {:?}", e),
}
let audit_retention_days = audit_log_retention_days().await;
@@ -2866,6 +2911,21 @@ pub async fn reload_retention_period_setting(conn: &Connection) {
}
}
+pub async fn reload_service_log_retention_secs_setting(conn: &Connection) {
+ match load_setting_value::(
+ conn,
+ SERVICE_LOG_RETENTION_SECS_SETTING,
+ "SERVICE_LOG_RETENTION_SECS",
+ DEFAULT_SERVICE_LOG_RETENTION_SECS,
+ |x| x,
+ )
+ .await
+ {
+ Ok(v) => windmill_common::set_service_log_retention_secs(v),
+ Err(e) => tracing::error!("Error reloading service log retention period: {:?}", e),
+ }
+}
+
pub async fn reload_audit_log_retention_days_setting(conn: &Connection) {
match load_setting_value::(
conn,
diff --git a/backend/windmill-api-settings/src/log_cleanup.rs b/backend/windmill-api-settings/src/log_cleanup.rs
index ecbc38fcb7..82080cb61c 100644
--- a/backend/windmill-api-settings/src/log_cleanup.rs
+++ b/backend/windmill-api-settings/src/log_cleanup.rs
@@ -32,7 +32,7 @@ use windmill_common::tracing_init::{LOGS_SERVICE, TMP_WINDMILL_LOGS_SERVICE};
use windmill_common::worker::WINDMILL_DIR;
use windmill_common::{
DB, INSTANCE_NAME, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES,
- JOB_RETENTION_SECS_OVERRIDES_LOADED, SERVICE_LOG_RETENTION_SECS,
+ JOB_RETENTION_SECS_OVERRIDES_LOADED,
};
use windmill_object_store::object_store_reexports::{
@@ -249,7 +249,7 @@ async fn cleanup_service_logs(
// Count candidates upfront for progress reporting.
let total: i64 = sqlx::query_scalar!(
"SELECT COUNT(*) FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval",
- SERVICE_LOG_RETENTION_SECS,
+ windmill_common::service_log_retention_secs(),
)
.fetch_one(db)
.await?
@@ -274,7 +274,7 @@ async fn cleanup_service_logs(
WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval
LIMIT $2
) RETURNING file_path, hostname",
- SERVICE_LOG_RETENTION_SECS,
+ windmill_common::service_log_retention_secs(),
SERVICE_LOG_BATCH,
)
.fetch_all(db)
@@ -680,9 +680,10 @@ async fn cleanup_s3_orphans(
) -> error::Result<()> {
let job_retention_secs = JOB_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed);
let now = Utc::now();
- // Service logs always have a retention (hardcoded SERVICE_LOG_RETENTION_SECS),
- // so we scan for service-log orphans regardless of JOB_RETENTION_SECS.
- let service_cutoff = now - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS);
+ // Service logs always have a retention, so we scan for service-log orphans regardless of
+ // JOB_RETENTION_SECS.
+ let service_cutoff =
+ now - chrono::Duration::seconds(windmill_common::service_log_retention_secs());
// Job-log orphans are only considered once past a job's effective retention window. That window
// is the instance one OR, for an override workspace (EE), its own — and jobs orphan their logs as
diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs
index b539c1c6b3..b1ad2edd01 100644
--- a/backend/windmill-common/src/global_settings.rs
+++ b/backend/windmill-common/src/global_settings.rs
@@ -15,6 +15,7 @@ pub const OAUTH_SETTING: &str = "oauths";
pub const AI_CONFIG_SETTING: &str = "ai_config";
pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs";
pub const RETENTION_PERIOD_SECS_OVERRIDES_SETTING: &str = "retention_period_secs_overrides";
+pub const SERVICE_LOG_RETENTION_SECS_SETTING: &str = "service_log_retention_secs";
/// Upper bound on how many per-workspace retention overrides may be configured. The periodic monitor
/// sweeps each override workspace in its own transaction every pass, so this keeps a pass bounded
/// (and the feature is a targeted escape hatch for a handful of special workspaces, not a bulk knob).
diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs
index 7b1ed407ab..559eba3bf9 100644
--- a/backend/windmill-common/src/indexer.rs
+++ b/backend/windmill-common/src/indexer.rs
@@ -94,6 +94,21 @@ pub async fn load_indexer_config(db: &DB) -> error::Result i64 {
+ let retention = crate::service_log_retention_secs();
+ if max_index_time_window_secs > 0 {
+ std::cmp::min(max_index_time_window_secs, retention)
+ } else {
+ retention
+ }
+}
+
pub fn get_env_var(env_var: &str) -> Option {
match std::env::var(env_var).map(|x| x.parse()) {
Ok(Ok(i)) => Some(i),
@@ -136,3 +151,60 @@ pub fn get_indexer_rates_from_env() -> TantivyIndexerSettings {
settings
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ // One test rather than several: both halves share the process-wide retention, and the
+ // setter half writes it, which parallel tests would race.
+ #[test]
+ fn retention_rejects_unusable_values_and_the_index_window_clamps_to_it() {
+ use crate::{
+ service_log_retention_secs, set_service_log_retention_secs,
+ DEFAULT_SERVICE_LOG_RETENTION_SECS,
+ };
+
+ // See `set_service_log_retention_secs` for why the two unusable directions land apart:
+ // too large keeps the intent by capping, non-positive cannot and falls back.
+ let rejected: Vec = [0, -1, i64::MIN]
+ .iter()
+ .map(|v| {
+ set_service_log_retention_secs(*v);
+ service_log_retention_secs()
+ })
+ .collect();
+ let capped: Vec = [i64::MAX, 60 * 60 * 24 * 365 * 101]
+ .iter()
+ .map(|v| {
+ set_service_log_retention_secs(*v);
+ service_log_retention_secs()
+ })
+ .collect();
+
+ set_service_log_retention_secs(60 * 60 * 24 * 3);
+ let retention = service_log_retention_secs();
+ let windows = [
+ // `0` disables the extra shrinking rather than lifting the ceiling — the trap that
+ // makes an unset setting look unbounded.
+ service_log_index_window_secs(0),
+ // Retention is the ceiling: the index cannot reach lines whose `log_file` row is gone.
+ service_log_index_window_secs(retention * 2),
+ service_log_index_window_secs(60),
+ ];
+ set_service_log_retention_secs(DEFAULT_SERVICE_LOG_RETENTION_SECS);
+
+ assert_eq!(
+ rejected,
+ vec![DEFAULT_SERVICE_LOG_RETENTION_SECS; 3],
+ "a value that would expire everything must fall back to the default"
+ );
+ assert_eq!(
+ capped,
+ vec![60 * 60 * 24 * 365 * 100; 2],
+ "an oversized value must cap, not shorten retention to the default"
+ );
+ assert_eq!(retention, 60 * 60 * 24 * 3);
+ assert_eq!(windows, [retention, retention, 60]);
+ }
+}
diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs
index cf0be2bbd8..f1784e9841 100644
--- a/backend/windmill-common/src/lib.rs
+++ b/backend/windmill-common/src/lib.rs
@@ -147,9 +147,53 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5;
pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev";
pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000;
-pub const SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs
+pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs
pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers";
+/// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower
+/// than an `i64`: `DateTime` subtraction panics past year 262143, and the `( s)::interval`
+/// the cleanup queries build overflows Postgres' microsecond field.
+const MAX_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100;
+
+/// Apply a configured service log retention, in seconds.
+///
+/// The only way into [`SERVICE_LOG_RETENTION_SECS`], so an unusable value can never reach a
+/// cutoff. The two unusable directions are not the same mistake and must not share a landing
+/// point: too large still says "keep these for a very long time", so it is capped and the
+/// intent survives, whereas falling back would delete logs the operator meant to keep. A
+/// non-positive value has no such reading — every cutoff is `now - retention`, so it lands at
+/// or after `now` and the next sweep expires the entire history, rows and object-storage files
+/// alike. Unlike job retention there is no "keep forever" spelling here, so `0` — what an
+/// operator types by analogy with it, and what the settings UI writes into a field that was
+/// merely focused — falls back to the default.
+pub fn set_service_log_retention_secs(configured: i64) {
+ let effective = if configured > MAX_SERVICE_LOG_RETENTION_SECS {
+ tracing::warn!(
+ "service log retention of {configured}s exceeds the maximum of \
+ {MAX_SERVICE_LOG_RETENTION_SECS}s, capping it there"
+ );
+ MAX_SERVICE_LOG_RETENTION_SECS
+ } else if configured >= 1 {
+ configured
+ } else {
+ tracing::warn!(
+ "service log retention of {configured}s would expire every service log, \
+ falling back to the default of {DEFAULT_SERVICE_LOG_RETENTION_SECS}s"
+ );
+ DEFAULT_SERVICE_LOG_RETENTION_SECS
+ };
+ SERVICE_LOG_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed);
+}
+
+/// How long a service log line stays retrievable, in seconds.
+///
+/// The outer bound on everything service-log: the `log_file` rows, the raw files in object
+/// storage, the columnar store queried by retrieval, and — through
+/// [`indexer::service_log_index_window_secs`] — the search index.
+pub fn service_log_retention_secs() -> i64 {
+ SERVICE_LOG_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed)
+}
+
/// Canonical form of a base URL, used as one of the inputs to the offline-license
/// instance hash (`compute_instance_hash`).
///
@@ -375,6 +419,10 @@ lazy_static::lazy_static! {
/// workspace configured before its override could be read.
pub static ref JOB_RETENTION_SECS_OVERRIDES_LOADED: AtomicBool = AtomicBool::new(false);
pub static ref AUDIT_LOG_RETENTION_DAYS: AtomicI64 = AtomicI64::new(0);
+ /// Private on purpose: [`set_service_log_retention_secs`] is the only writer, so a value that
+ /// would expire every service log cannot reach a cutoff. Read it with
+ /// [`service_log_retention_secs`].
+ static ref SERVICE_LOG_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_SERVICE_LOG_RETENTION_SECS);
pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false);
diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte
index 91e00d8c71..4a0bd58715 100644
--- a/frontend/src/lib/components/InstanceSettings.svelte
+++ b/frontend/src/lib/components/InstanceSettings.svelte
@@ -1143,6 +1143,32 @@
description="Configure default timeouts and retention policies for job execution."
link="https://www.windmill.dev/docs/advanced/instance_settings#jobs"
/>
+ {:else if category == 'Service logs'}
+
+ {#if !$values['object_store_cache_config']}
+
+
+ Instance object storage is not configured, so every server and worker keeps its log
+ files on its own disk. This page lists what each host wrote, but can only open the
+ files belonging to the replica serving the request — another host's are listed and
+ not readable — and a host's files go with it when it is replaced. Retention below
+ still governs the entries in the database and the files on disk.
+
+
+ {:else if !$enterpriseLicense}
+
+
+ Log files are uploaded to instance object storage, and the indexer that would ingest
+ them into the columnar store and delete each one afterwards is an enterprise
+ feature. Retention below expires the database entries and the local files; the
+ uploaded copies are only removed when Delete logs from s3 periodically is on
+ under Object Storage.
+
+
+ {/if}
{:else if category == 'Object Storage'}
= {
triggersRestart: true
}
],
+ 'Service logs': [
+ {
+ label: 'Retention in secs',
+ key: 'service_log_retention_secs',
+ description:
+ 'How long a service log is kept, across every copy of it: the entry in the database, the file on the disk of the process that wrote it, and — once instance object storage is configured and the indexer has ingested it — its line in the columnar store that search and the log viewer read. Search reaches back at most this far, and less when the indexer time window under Indexer is shorter. Defaults to 14 days. There is no keep-forever setting here — leave it empty for the default.',
+ fieldType: 'seconds',
+ storage: 'setting',
+ cloudonly: false,
+ error:
+ 'Service log retention must be between 1 second and 100 years — leave it empty for the default',
+ isValid: (value: any) =>
+ value == undefined ||
+ (typeof value === 'number' && value > 0 && value <= 60 * 60 * 24 * 365 * 100)
+ }
+ ],
+
Indexer: [
{
label: '',
@@ -1173,6 +1190,12 @@ export const instanceSettingsNavigationGroups = [
aiDescription: 'Instance OTEL/Prometheus settings',
isEE: true
},
+ {
+ id: 'service_logs',
+ label: 'Service logs',
+ aiId: 'instance-settings-service-logs',
+ aiDescription: 'Service log retention settings'
+ },
{
id: 'indexer',
label: 'Indexer',
@@ -1256,6 +1279,7 @@ export const tabToCategoryMap: Record = {
webhooks: 'Webhooks',
otel_prom: 'OTEL/Prom',
indexer: 'Indexer',
+ service_logs: 'Service logs',
telemetry: 'Telemetry',
secret_storage: 'Secret Storage',
object_storage: 'Object Storage',
@@ -1291,6 +1315,7 @@ export const categoryToTabMap: Record = {
Webhooks: 'webhooks',
'OTEL/Prom': 'otel_prom',
Indexer: 'indexer',
+ 'Service logs': 'service_logs',
Telemetry: 'telemetry',
'Secret Storage': 'secret_storage',
'Object Storage': 'object_storage',
From 7639d83a4254fb5728ad1c7f200c6c197fcbbd4b Mon Sep 17 00:00:00 2001
From: Ruben Fiszel
Date: Sat, 29 Aug 2026 19:45:12 +0200
Subject: [PATCH 016/310] chore: bump ee-repo-ref to the merged EE main
(#10896)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
windmill-ee-private#756 was squash-merged, so the commit ee-repo-ref names is not on EE main
and the branch carrying it is gone. The content is identical, so nothing builds differently —
but a dangling ref is one garbage collection away from an EE build that cannot fetch what it
is pinned to.
Claude-Session: https://claude.ai/code/session_01WsnpNSM6K3oyjwntRwJtVN
Co-authored-by: Claude Opus 5 (1M context)
---
backend/ee-repo-ref.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index 6f56f0ab9a..1ca9152918 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-e6483ff5a289521405912d95be5c5ba064bedb38
+f3c923975012e5e499fb07d8b390b61808bb8374
From d91ee4614a70f20a194f47e190327129f499ec63 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel
Date: Sun, 30 Aug 2026 07:16:09 +0200
Subject: [PATCH 017/310] feat: day-partition the service log index and expire
whole chunks (#10893)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat: day-partition the service log index and expire whole chunks
The service log index becomes one tantivy index per UTC day. The substance is
in windmill-ee-private#753; this side carries the EE ref and moves the log
indexer writer instead of cloning it, because sealing a chunk takes sole
ownership of its tantivy writer.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX
* fix: do not adopt the superseded watermark after an explicit index clear
A clear asks for the retention window to be read again, and a watermark says
it already has been — and the v3 copy in object storage is kept for rollback,
so it outlives the local one the clear removes. Both copies of that watermark
are now read and the newer wins, for the same reason the v4 one is taken from
the store when it is ahead: a replica that lost the lock keeps a local file
frozen where it stopped while the store went on.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX
* fix: delete a day's raw files at its checkpoint, and rebuild whole days
Bumps the EE ref for windmill-ee-private#753.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX
* fix: make an interrupted rebuild detectable, and pin the rebuild floor
Bumps the EE ref for windmill-ee-private#753.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX
* fix: keep the rebuild marker in the object store, not on local disk
Bumps the EE ref for windmill-ee-private#753.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX
* fix: two more routes to a partial index being accepted as complete
Bumps the EE ref for windmill-ee-private#753.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX
* fix: trust a local chunk only when the tracker vouches for it
Bumps the EE ref for windmill-ee-private#753.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX
* chore: condense the stale-chunk guard's doc to the four-line limit
Bumps the EE ref for windmill-ee-private#753.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01EeUmYWCJeaaHutiHLfZBQX
* chore: update ee-repo-ref to 17ef439b087b400889ff19109be9d2c810142278
This commit updates the EE repository reference after PR #753 was merged in windmill-ee-private.
Previous ee-repo-ref: 3e79901b4742906d2285dd943e24fac0f735f199
New ee-repo-ref: 17ef439b087b400889ff19109be9d2c810142278
Automated by sync-ee-ref workflow.
---------
Co-authored-by: Claude Opus 5 (1M context)
Co-authored-by: windmill-internal-app[bot]
---
backend/ee-repo-ref.txt | 2 +-
backend/src/main.rs | 6 ++++--
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index 1ca9152918..ee579195bc 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-f3c923975012e5e499fb07d8b390b61808bb8374
+17ef439b087b400889ff19109be9d2c810142278
diff --git a/backend/src/main.rs b/backend/src/main.rs
index c782e1ad97..fa88cf6cdc 100644
--- a/backend/src/main.rs
+++ b/backend/src/main.rs
@@ -1264,10 +1264,12 @@ Windmill Community Edition {GIT_VERSION}
#[cfg(all(feature = "tantivy", feature = "parquet"))]
let log_indexer_f = {
let log_indexer_rx = killpill_rx.resubscribe();
- let log_index_writer2 = log_index_writer.clone();
+ // Moved, not cloned: sealing a chunk takes sole ownership of its
+ // tantivy writer, which a second live handle would silently prevent.
+ let moved_log_index_writer = log_index_writer;
async {
if let Some(db) = conn.as_sql() {
- if let Some(log_index_writer) = log_index_writer2 {
+ if let Some(log_index_writer) = moved_log_index_writer {
windmill_indexer::service_logs_oss::run_indexer(
db.clone(),
log_index_writer,
From ac56586c0e56d4022761d3c80306a03d57f8bfcb Mon Sep 17 00:00:00 2001
From: Ruben Fiszel
Date: Sun, 30 Aug 2026 07:40:34 +0200
Subject: [PATCH 018/310] fix: correct the service log ingest flush boundary
(#10898)
Claude-Session: https://claude.ai/code/session_014KnE8my2okxQGCx47cWMjf
Co-authored-by: Claude Opus 5 (1M context)
---
backend/ee-repo-ref.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index ee579195bc..5613e1173c 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-17ef439b087b400889ff19109be9d2c810142278
+25d911019aebd3779bb9461a5faab4c20e2f9cf0
From 2fb790338d130c9ce64afe70c8f9fd0fd091219a Mon Sep 17 00:00:00 2001
From: Ruben Fiszel
Date: Mon, 31 Aug 2026 08:48:27 +0200
Subject: [PATCH 019/310] upgrade argon2 to 0.6 and migrate the password
hashing API (#10902)
* fix: upgrade argon2 to 0.6 and migrate the password hashing API
* test: pin that an unparseable stored hash reads as a failed login
* chore: update ee-repo-ref to 58738c39ac41d57917bbd9400318704763d997f7
This commit updates the EE repository reference after PR #759 was merged in windmill-ee-private.
Previous ee-repo-ref: 02a89fc4d27e49a494112fa91a8812e3ee4fb8a6
New ee-repo-ref: 58738c39ac41d57917bbd9400318704763d997f7
Automated by sync-ee-ref workflow.
---------
Co-authored-by: windmill-internal-app[bot]
---
backend/Cargo.lock | 100 +++++++++++++++---
backend/Cargo.toml | 8 +-
backend/ee-repo-ref.txt | 2 +-
.../tests/users.rs | 23 ++--
backend/windmill-api-users/src/users.rs | 26 ++++-
5 files changed, 128 insertions(+), 31 deletions(-)
diff --git a/backend/Cargo.lock b/backend/Cargo.lock
index 7c8d40ad05..9bfefb16d9 100644
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -23,7 +23,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
dependencies = [
- "crypto-common",
+ "crypto-common 0.1.7",
"generic-array",
]
@@ -261,13 +261,13 @@ dependencies = [
[[package]]
name = "argon2"
-version = "0.5.3"
+version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072"
+checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c"
dependencies = [
"base64ct",
- "blake2",
- "cpufeatures 0.2.17",
+ "blake2 0.11.0",
+ "cpufeatures 0.3.1",
"password-hash",
]
@@ -1824,6 +1824,15 @@ dependencies = [
"digest 0.10.7",
]
+[[package]]
+name = "blake2"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f"
+dependencies = [
+ "digest 0.11.3",
+]
+
[[package]]
name = "blake3"
version = "1.8.7"
@@ -1856,6 +1865,15 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "block-buffer"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
+dependencies = [
+ "hybrid-array",
+]
+
[[package]]
name = "block-modes"
version = "0.8.1"
@@ -2390,7 +2408,7 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
- "crypto-common",
+ "crypto-common 0.1.7",
"inout",
]
@@ -2454,6 +2472,12 @@ dependencies = [
"cc",
]
+[[package]]
+name = "cmov"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
+
[[package]]
name = "cms"
version = "0.2.3"
@@ -2818,6 +2842,15 @@ dependencies = [
"typenum",
]
+[[package]]
+name = "crypto-common"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
+dependencies = [
+ "hybrid-array",
+]
+
[[package]]
name = "csv"
version = "1.4.0"
@@ -2848,6 +2881,15 @@ dependencies = [
"cipher 0.4.4",
]
+[[package]]
+name = "ctutils"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e"
+dependencies = [
+ "cmov",
+]
+
[[package]]
name = "curl-sys"
version = "0.4.90+curl-8.21.0"
@@ -3413,7 +3455,7 @@ dependencies = [
"arrow",
"arrow-buffer",
"base64 0.22.1",
- "blake2",
+ "blake2 0.10.6",
"blake3",
"chrono",
"datafusion-common",
@@ -4455,10 +4497,21 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"const-oid",
- "crypto-common",
+ "crypto-common 0.1.7",
"subtle",
]
+[[package]]
+name = "digest"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
+dependencies = [
+ "block-buffer 0.12.1",
+ "crypto-common 0.2.2",
+ "ctutils",
+]
+
[[package]]
name = "dirs"
version = "4.0.0"
@@ -6139,6 +6192,15 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
+[[package]]
+name = "hybrid-array"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b"
+dependencies = [
+ "typenum",
+]
+
[[package]]
name = "hyper"
version = "0.14.32"
@@ -8924,13 +8986,12 @@ dependencies = [
[[package]]
name = "password-hash"
-version = "0.5.0"
+version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166"
+checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b"
dependencies = [
- "base64ct",
- "rand_core 0.6.4",
- "subtle",
+ "getrandom 0.4.3",
+ "phc",
]
[[package]]
@@ -9074,6 +9135,17 @@ dependencies = [
"phf 0.11.3",
]
+[[package]]
+name = "phc"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892"
+dependencies = [
+ "base64ct",
+ "ctutils",
+ "getrandom 0.4.3",
+]
+
[[package]]
name = "phf"
version = "0.11.3"
@@ -14145,7 +14217,7 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
dependencies = [
- "crypto-common",
+ "crypto-common 0.1.7",
"subtle",
]
diff --git a/backend/Cargo.toml b/backend/Cargo.toml
index 3f614c9388..86940805fc 100644
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -477,10 +477,10 @@ rust-embed = { version = "^6", features = ["interpolate-folder-path"] }
mime_guess = "^2"
hex = "^0"
sql-builder = "^3"
-# Pinned: `^0` floats to 0.6, which moved `password_hash::SaltString`, put
-# `rand_core` behind a feature and changed `hash_password`'s signature —
-# users_ee.rs is written against 0.5 and does not compile otherwise.
-argon2 = "0.5"
+# Minor-pinned rather than the `^0` used elsewhere in this file: argon2's 0.x
+# minors are API-breaking (0.6 moved `SaltString` into `phc`, put `rand_core`
+# behind a feature and changed `hash_password`), so a float breaks the build.
+argon2 = "0.6"
quick_cache = "^0"
rand = "=0.9.0"
rand_core = { version = "^0", features = ["std"] }
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index 5613e1173c..8e49c42d98 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-25d911019aebd3779bb9461a5faab4c20e2f9cf0
+58738c39ac41d57917bbd9400318704763d997f7
diff --git a/backend/windmill-api-integration-tests/tests/users.rs b/backend/windmill-api-integration-tests/tests/users.rs
index fb64d9eb77..bec59ac45a 100644
--- a/backend/windmill-api-integration-tests/tests/users.rs
+++ b/backend/windmill-api-integration-tests/tests/users.rs
@@ -308,14 +308,17 @@ async fn test_user_endpoints(db: Pool) -> anyhow::Result<()> {
let auth_base = format!("http://localhost:{port}/api/auth");
// --- login (will fail: password hash in fixture is fake) ---
+ // An unparseable stored hash must read as a failed login, not as a server error
+ // relaying the hash parser's message to an unauthenticated caller.
let resp = client()
.post(format!("{auth_base}/login"))
.json(&json!({"email": "test@windmill.dev", "password": "wrong-password"}))
.send()
.await
.unwrap();
- assert!(
- resp.status() == 400 || resp.status() == 401 || resp.status() == 500,
+ assert_eq!(
+ resp.status(),
+ 400,
"login: unexpected status {}",
resp.status()
);
@@ -804,12 +807,16 @@ async fn test_change_user_email_leaves_group_identities(db: Pool) -> a
let server = ApiServer::start(db.clone()).await?;
let global_base = format!("http://localhost:{}/api/users", server.addr.port());
- sqlx::query!("UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'")
- .execute(&db)
- .await?;
- sqlx::query!("UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'")
- .execute(&db)
- .await?;
+ sqlx::query!(
+ "UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'"
+ )
+ .execute(&db)
+ .await?;
+ sqlx::query!(
+ "UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'"
+ )
+ .execute(&db)
+ .await?;
sqlx::query!(
"INSERT INTO group_(workspace_id, name, summary, extra_perms) VALUES ('test-workspace', 'ops', '', '{}')"
)
diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs
index 8ea3bd0995..8daa414b46 100644
--- a/backend/windmill-api-users/src/users.rs
+++ b/backend/windmill-api-users/src/users.rs
@@ -19,7 +19,7 @@ use windmill_api_auth::ApiAuthed;
pub use windmill_api_auth::Tokened;
-use argon2::{Argon2, PasswordHash, PasswordVerifier};
+use argon2::{Argon2, PasswordVerifier};
use axum::{
extract::{Extension, Path, Query},
response::{IntoResponse, Response},
@@ -2680,10 +2680,8 @@ async fn login(
.await?;
if let Some((email, hash, super_admin)) = email_w_h {
- let parsed_hash =
- PasswordHash::new(&hash).map_err(|e| Error::internal_err(e.to_string()))?;
if argon2
- .verify_password(password.as_bytes(), &parsed_hash)
+ .verify_password(password.as_bytes(), hash.as_str())
.is_err()
{
audit_log(
@@ -3710,3 +3708,23 @@ async fn request_password_reset(
}
// NOTE: reset_password is in windmill-api (depends on users_oss::hash_password EE dispatch)
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Stored hashes outlive the hashing crate: every instance still holds hashes minted by
+ /// older argon2 releases, and an upgrade that stopped reading them locks their users out.
+ #[test]
+ fn verifies_a_hash_minted_by_an_older_argon2() {
+ // The seeded admin hash from migration 20220508150023, m=4096,t=3,p=1.
+ let seeded = "$argon2id$v=19$m=4096,t=3,p=1$oLJo/lPn/gezXCuFOEyaNw$i0T2tCkw3xUFsrBIKZwr8jVNHlIfoxQe+HfDnLtd12I";
+
+ assert!(Argon2::default()
+ .verify_password(b"changeme", seeded)
+ .is_ok());
+ assert!(Argon2::default()
+ .verify_password(b"not-the-password", seeded)
+ .is_err());
+ }
+}
From aa4a6ffd66813010a79c07741b01a984ed4e7df6 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel
Date: Mon, 31 Aug 2026 14:06:29 +0200
Subject: [PATCH 020/310] fix: track outstanding service log files on the rows
themselves (#10894)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* fix: track outstanding service log files on the rows themselves
Adds `log_file.indexed_at` so the service log ingest can read outstanding rows
instead of walking a cursor over `log_ts`. A row registered after the pass had
gone by its minute was skipped for good, and no ordering fixes that — an arrival
sequence fails the same way, since a row can take a lower value and commit after
a higher one has moved the cursor past it.
The migration marks existing rows with a sentinel; the first pass returns the
ones the old cursor had not reached to the queue.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01EPAP96jJNYpPQ8bpxZcU1C
* [ee] refactor: drop the claim/confirm phase from the service log ingest queue
Two states are enough: a row is outstanding or it is marked. The migration no
longer creates the index for the claim sentinel, and the sqlx cache loses the
two queries the event-time cursor used.
* [ee] fix: make re-indexing a service log file idempotent
Corrects the `init_last_log_file_sent` note: a rewritten row keeps the
`indexed_at` it had, so one the indexers already took is not offered again.
* [ee] fix: let a rebuild take the rows it covered out of the ingest queue
Adds the query that releases them; the index layout stays v4.
* [ee] fix: index the lookup a rebuild releases rows by
A rebuild takes rows out of the queue by the file it read out of the store, which
is the one lookup that arrives without a `log_ts`. The primary key is
`(hostname, log_ts)`, so nothing covered it and each batch scanned every
outstanding row — worst in exactly the state a rebuild follows. Verified at 50k
outstanding rows: sequential scan becomes an index scan.
Also records `log_file.indexed_at` in the schema reference.
* [ee] fix: treat a state handed back without its line count as behind
* [ee] fix: give the converted state a line count
* [ee] fix: keep the converted cursor from being rewound by the rebuild
* [ee] fix: inherit the legacy cursor from one source, not field by field
* [ee] fix: count a file's lines against the buffer before reading it
* [ee] fix: bound the row buffer on what it holds, not on reported counts
* [ee] fix: settle the upgrade from the store rather than from event time
* [ee] docs: describe the conversion's second half as it now works
* [ee] refactor: settle the upgrade with one rebuild instead of reconciling
The migration records existing rows as done rather than marking them with a
sentinel: the indexer puts back what the old cursor had not reached on its first
pass, which is the only place that cursor's position is known.
* [ee] fix: repair the rows the old cursor skipped instead of recording them as done
The migration marks pre-existing rows with a sentinel again, so the indexer can
tell them from rows registered since and put the window's worth back on the queue.
* [ee] fix: keep a source file whole in one partition
* [ee] revert the file-atomic partition change
* [ee] fix: dedupe the public reads, and repair an index without a cursor
* [ee] fix: repair an index whose cursor is gone, and keep what the repair found
* [ee] fix: seed a pass from both axes of what a rebuild recovered
* [ee] fix: settle the cursor on what the store holds, not on what was read
* [ee] fix: an empty rebuild must not claim ground it has not covered
* [ee] test: pin the cursor a rebuild settles on
* chore: update ee-repo-ref to bc0c7051585194474078b6c1941a3fb73893d9e5
This commit updates the EE repository reference after PR #755 was merged in windmill-ee-private.
Previous ee-repo-ref: 328f5a90afeae9c683bf3294f0d9eb293a3e1a92
New ee-repo-ref: bc0c7051585194474078b6c1941a3fb73893d9e5
Automated by sync-ee-ref workflow.
---------
Co-authored-by: Claude Opus 5 (1M context)
Co-authored-by: windmill-internal-app[bot]
---
AGENTS.md | 9 +++
...463a0e7b5248c226ee92d95df94c3099cc400.json | 15 +++++
...b14b842f595f68dd885248abaaeabd2d0bf1.json} | 5 +-
...4633c1531170ded1fdf09114718b941f5e1db.json | 65 -------------------
...cdc85389c0629847d8fabb2ad0aa043957b2f.json | 22 +++++++
...0c06924719189bec8e5c19866db8d0b87df5e.json | 15 +++++
...fd56e616ee79d895b7dcc37ad9442789e1574.json | 22 +++++++
...53d23ec2af084b2f93da24c920532c1916384.json | 6 +-
backend/ee-repo-ref.txt | 2 +-
...0260830085453_log_file_indexed_at.down.sql | 4 ++
.../20260830085453_log_file_indexed_at.up.sql | 35 ++++++++++
backend/src/monitor.rs | 13 ++--
backend/summarized_schema.txt | 2 +-
13 files changed, 136 insertions(+), 79 deletions(-)
create mode 100644 backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json
rename backend/.sqlx/{query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json => query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json} (86%)
delete mode 100644 backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json
create mode 100644 backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json
create mode 100644 backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json
create mode 100644 backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json
create mode 100644 backend/migrations/20260830085453_log_file_indexed_at.down.sql
create mode 100644 backend/migrations/20260830085453_log_file_indexed_at.up.sql
diff --git a/AGENTS.md b/AGENTS.md
index 47919cfeda..71c59b479d 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -165,6 +165,15 @@ $NAV --root backend callees "X" # what does X call?
- Search for existing code to reuse before writing new code
- Follow established patterns in the codebase
- Keep changes focused — don't refactor beyond what's asked
+- **A simpler design found late is still the design.** Work already spent is not an argument
+ for a shape, and neither is a clean review round, a passing suite, or a long PR thread. The
+ signal to stop and re-derive rather than patch again is a change that keeps growing to defend
+ its own structure: each review finding fixing an assumption the previous fix broke, the same
+ class of bug reappearing somewhere new, or most of the diff being consequences of one early
+ choice rather than the thing you set out to do. When that happens, say plainly what the
+ simpler design is and what switching costs — a migration, a review cycle restarted from zero,
+ work discarded — and let the user decide. Do not keep paying down the harder one because it
+ is nearly finished, and do not present the accumulated cost as a reason to continue.
- **Ship only the tests the PR needs.** A committed test must pin behavior a future change could plausibly break, and be the smallest setup that exercises the new logic. While developing, write as many exhaustive tests and do as much manual testing as you need to convince yourself the change works — then remove that scaffolding before marking the PR ready, keeping only the essential regression guard(s). A test that merely re-exercises pre-existing behavior, or needs elaborate fixtures to assert something trivial, is scaffolding: delete it. If nothing meaningful is left to guard, ship no test rather than a ceremonial one.
- **Comments record constraints, not narration.** Write a comment only for what the code can't show: why a non-obvious approach is required, what breaks if it's "simplified" away. State each invariant once, at the place where someone would break it, in ≤4 lines. Don't describe what the next line does, don't repeat the same rationale at multiple sites, and don't address the PR reviewer (justifying a change belongs in the PR description, not the code). Reference nothing ephemeral — no numbered steps from your dev flow, no "the poller / the test does X" scaffolding, no transient state that won't exist for the next reader; keep only the essential, durable rationale. Describe the code as it is, never its drafting history: "we no longer do X", "unchanged behavior", "instead of the previous approach" are meaningless to a reader who never saw the earlier iteration — before finishing, reread your comments as if the current state is the only state that ever existed.
- **Never attribute work to a specific customer, account, or "requested by a customer" in repo-tracked content** (PR descriptions, commit messages, code comments, docs). Describe changes by their technical motivation instead.
diff --git a/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json b/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json
new file mode 100644
index 0000000000..3713af1486
--- /dev/null
+++ b/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::text[]) AS c(hostname, file_path)\n WHERE log_file.indexed_at IS NULL\n AND log_file.hostname = c.hostname\n AND log_file.file_path = c.file_path",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "TextArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400"
+}
diff --git a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json b/backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json
similarity index 86%
rename from backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json
rename to backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json
index ba9f3814e8..13d1c9b1e8 100644
--- a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json
+++ b/backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > $1\n ORDER BY log_ts ASC LIMIT $2",
+ "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE indexed_at IS NULL\n ORDER BY log_ts ASC, hostname ASC LIMIT $1",
"describe": {
"columns": [
{
@@ -46,7 +46,6 @@
],
"parameters": {
"Left": [
- "Timestamp",
"Int8"
]
},
@@ -61,5 +60,5 @@
true
]
},
- "hash": "b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a"
+ "hash": "6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1"
}
diff --git a/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json b/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json
deleted file mode 100644
index d0b96444ee..0000000000
--- a/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json
+++ /dev/null
@@ -1,65 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > NOW() - make_interval(secs => $1)\n ORDER BY log_ts ASC LIMIT $2",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "hostname",
- "type_info": "Varchar"
- },
- {
- "ordinal": 1,
- "name": "mode",
- "type_info": "Text"
- },
- {
- "ordinal": 2,
- "name": "worker_group",
- "type_info": "Varchar"
- },
- {
- "ordinal": 3,
- "name": "log_ts",
- "type_info": "Timestamp"
- },
- {
- "ordinal": 4,
- "name": "file_path",
- "type_info": "Varchar"
- },
- {
- "ordinal": 5,
- "name": "ok_lines",
- "type_info": "Int8"
- },
- {
- "ordinal": 6,
- "name": "err_lines",
- "type_info": "Int8"
- },
- {
- "ordinal": 7,
- "name": "json_fmt",
- "type_info": "Bool"
- }
- ],
- "parameters": {
- "Left": [
- "Float8",
- "Int8"
- ]
- },
- "nullable": [
- false,
- null,
- true,
- false,
- false,
- true,
- true,
- true
- ]
- },
- "hash": "8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db"
-}
diff --git a/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json b/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json
new file mode 100644
index 0000000000..96745f9485
--- /dev/null
+++ b/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "WITH moved AS (\n UPDATE log_file SET indexed_at = CASE\n WHEN log_ts > NOW() - make_interval(secs => $1) THEN NULL\n ELSE now() END\n WHERE indexed_at = 'epoch' RETURNING 1)\n SELECT count(*) AS \"n!\" FROM moved",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "n!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Float8"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f"
+}
diff --git a/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json b/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json
new file mode 100644
index 0000000000..6739d80033
--- /dev/null
+++ b/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::timestamp[]) AS c(hostname, log_ts)\n WHERE log_file.hostname = c.hostname AND log_file.log_ts = c.log_ts",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "TimestampArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e"
+}
diff --git a/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json b/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json
new file mode 100644
index 0000000000..a83c34e0db
--- /dev/null
+++ b/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "WITH retired AS (\n UPDATE log_file SET indexed_at = now()\n WHERE indexed_at IS NULL\n AND log_ts <= NOW() - make_interval(secs => $1) RETURNING 1)\n SELECT count(*) AS \"n!\" FROM retired",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "n!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Float8"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574"
+}
diff --git a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json
index b336210daf..b892061f56 100644
--- a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json
+++ b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json
@@ -98,12 +98,12 @@
null,
null,
null,
- true,
+ false,
null,
null,
null,
- true,
- true
+ false,
+ false
]
},
"hash": "b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384"
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index 8e49c42d98..693921efc6 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-58738c39ac41d57917bbd9400318704763d997f7
+bc0c7051585194474078b6c1941a3fb73893d9e5
diff --git a/backend/migrations/20260830085453_log_file_indexed_at.down.sql b/backend/migrations/20260830085453_log_file_indexed_at.down.sql
new file mode 100644
index 0000000000..ac3c92f3eb
--- /dev/null
+++ b/backend/migrations/20260830085453_log_file_indexed_at.down.sql
@@ -0,0 +1,4 @@
+DROP INDEX IF EXISTS index_log_file_premigration;
+DROP INDEX IF EXISTS index_log_file_pending_path;
+DROP INDEX IF EXISTS index_log_file_pending;
+ALTER TABLE log_file DROP COLUMN IF EXISTS indexed_at;
diff --git a/backend/migrations/20260830085453_log_file_indexed_at.up.sql b/backend/migrations/20260830085453_log_file_indexed_at.up.sql
new file mode 100644
index 0000000000..3421a5082a
--- /dev/null
+++ b/backend/migrations/20260830085453_log_file_indexed_at.up.sql
@@ -0,0 +1,35 @@
+-- The service log ingest walked `log_file` with a cursor over `log_ts`, which is when a line
+-- was written rather than when its row appeared. Rows do not arrive in that order — an upload
+-- retried after a failure, a host that has just started, a batch the row limit cut mid-minute —
+-- and a row that becomes visible behind the cursor is never read: it stays in `log_file` and its
+-- lines stay out of search until retention drops them.
+--
+-- No ordering fixes this. A cursor over arrival order fails the same way, because `nextval` is
+-- allocated before its INSERT commits: a row can be assigned a lower value and commit after a
+-- higher one has already moved the cursor past it. Which rows are outstanding is a property of
+-- the rows, so it is recorded on them.
+ALTER TABLE log_file ADD COLUMN indexed_at TIMESTAMPTZ;
+
+-- Rows that already existed are marked, not queued: on a 14-day window most were ingested long
+-- ago and their raw files are gone. A sentinel rather than a timestamp, because the indexer has to
+-- tell them apart from rows registered since — those start NULL — and it puts the window's worth of
+-- them back on the queue on its first pass, keeping only what the columnar store can vouch for.
+--
+-- Not split here on the cursor the old ingest had reached. Below that cursor sits every row it
+-- skipped, which is the loss this migration exists to stop; recording those as done would carry the
+-- bug into its own fix.
+UPDATE log_file SET indexed_at = 'epoch' WHERE indexed_at IS NULL;
+
+-- The work queue, and the only index the ingest query needs: outstanding rows are a small
+-- fraction of the table, so this stays proportional to what is left to do rather than to the
+-- retention window.
+CREATE INDEX index_log_file_pending ON log_file (log_ts) WHERE indexed_at IS NULL;
+
+-- A rebuild takes rows out of the queue by the file it just read out of the store, which is
+-- the one lookup that arrives without a `log_ts`: the primary key is `(hostname, log_ts)`, so
+-- nothing else covers it and each batch would scan every outstanding row instead.
+CREATE INDEX index_log_file_pending_path ON log_file (hostname, file_path) WHERE indexed_at IS NULL;
+
+-- Reached once per pass while pre-migration rows survive, and never again after the first
+-- conversion clears them.
+CREATE INDEX index_log_file_premigration ON log_file (log_ts) WHERE indexed_at = 'epoch';
diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs
index c0748a3798..ce09cebe71 100644
--- a/backend/src/monitor.rs
+++ b/backend/src/monitor.rs
@@ -1358,9 +1358,9 @@ fn last_log_file_sent() -> Option {
/// the file that was still open and the appender reopens that minute in append mode,
/// so a restart inside it would otherwise strand everything written afterwards.
///
-/// A row rewritten this way restores the object and sums the counters, but whether the
-/// indexers read it again depends on their single `log_ts >` cursor, which is not
-/// per-hostname: a minute at or below it stays out of search until it is re-indexed.
+/// A row rewritten this way restores the object and sums the counters, but it keeps the
+/// `indexed_at` it already had, so one the indexers have taken is not offered again and
+/// the lines added by the rewrite stay out of search.
async fn init_last_log_file_sent(conn: &Connection, hostname: &str) {
let Some(db) = conn.as_sql() else {
return;
@@ -1406,9 +1406,10 @@ async fn send_log_files_to_object_store(
if ts < retention_cutoff {
continue;
}
- // Stop at the first failure rather than moving on: both indexers walk
- // `log_file` with a `log_ts > watermark` cursor, so a row that lands after
- // a newer one is never picked up.
+ // Stop at the first failure rather than moving on, so a file is never
+ // registered before an older one that has not made it to the store yet.
+ // The indexers do not depend on that ordering — every row is offered until
+ // it is marked — but a gap here would still be visible while it lasts.
if !send_log_file_to_object_store(hostname, mode, worker_group, conn, &file_name, ts).await
{
break;
diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt
index a9865cebd3..607ef99e0c 100644
--- a/backend/summarized_schema.txt
+++ b/backend/summarized_schema.txt
@@ -128,7 +128,7 @@ job_stats: workspace_id(char), job_id(uuid), metric_id(char), metric_name(char),
kafka_pending_commits: id(bigint), workspace_id(char), kafka_trigger_path(char), topic(char), partition(int), offset(bigint), created_at(ts)
FK: (workspace_id, kafka_trigger_path) -> kafka_trigger(workspace_id, path)
kafka_trigger: path(char), kafka_resource_path(char), topics(char), group_id(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), filters(jsonb[]), auto_commit(bool), labels(text[])
-log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool)
+log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool), indexed_at(ts)
macro_definition: workspace_id(char), name(char), provider_path(char), params(text), body(text), is_table_macro(bool), created_at(ts)
FK: (workspace_id) -> workspace(id)
macro_usage: workspace_id(char), consumer_path(char), macro_name(char)
From 66123f3a9b8978c0084b02f50b44cffba125a13a Mon Sep 17 00:00:00 2001
From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com>
Date: Mon, 31 Aug 2026 15:26:08 +0200
Subject: [PATCH 021/310] feat: add --keep-deleted flag to wmill sync pull and
push (#10878)
* feat: add --keep-deleted flag to wmill sync pull and push
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01JMfhFSZQJJRLPrfkug6VoK
* fix: address review findings on --keep-deleted
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_01JMfhFSZQJJRLPrfkug6VoK
---------
Co-authored-by: Claude Opus 5 (1M context)
---
cli/src/commands/shared_ui.ts | 152 +++++++++++++-----
cli/src/commands/sync/sync.ts | 94 +++++++++--
cli/src/guidance/skills.gen.ts | 2 +
cli/test/shared_ui_diff_unit.test.ts | 61 ++++++-
cli/test/sync_pull_push.test.ts | 99 ++++++++++++
.../auto-generated/cli/cli-commands.md | 2 +
system_prompts/auto-generated/prompts.ts | 2 +
.../skills/cli-commands/SKILL.md | 2 +
8 files changed, 353 insertions(+), 61 deletions(-)
diff --git a/cli/src/commands/shared_ui.ts b/cli/src/commands/shared_ui.ts
index 3c01870fd1..f8e2965994 100644
--- a/cli/src/commands/shared_ui.ts
+++ b/cli/src/commands/shared_ui.ts
@@ -26,37 +26,33 @@ async function readDirRecursive(
return out;
}
+/** The workspace's shared UI store, or undefined when it cannot be read. */
+async function fetchSharedUi(
+ workspace: string,
+): Promise | undefined> {
+ try {
+ const got = await wmill.getSharedUi({ workspace });
+ return got.files ?? {};
+ } catch {
+ return undefined;
+ }
+}
+
export type SharedUiChange =
| { type: "added"; path: string }
| { type: "edited"; path: string; before: string; after: string }
| { type: "deleted"; path: string };
/**
- * Diff the local /ui/ folder against the workspace's shared UI store in
- * the push direction (local -> remote), returning entries whose `path` is
- * prefixed with `ui/`. This is the same comparison pushSharedUi applies, so the
- * dry-run preview and the real push never diverge.
- *
- * Mirrors pushSharedUi's no-op: with no local ui/ folder there is nothing to
- * push, so the apply is a no-op and the preview must be empty (even when the
- * remote store is non-empty) to avoid phantom diffs the apply won't perform.
+ * The push-direction diff (local -> remote) of two already-read file maps.
+ * Split out so the preview and the push itself compare identically off one
+ * read of each side.
*/
-export async function diffSharedUi(workspace: string): Promise {
- const localDir = path.join(process.cwd(), SHARED_UI_DIR);
- if (!fs.existsSync(localDir)) {
- return [];
- }
- const files = await readDirRecursive(localDir);
-
- let remote: Record = {};
- try {
- const got = await wmill.getSharedUi({ workspace });
- remote = got.files ?? {};
- } catch {
- // If endpoint missing or unauthorized, treat remote as empty (the push
- // would attempt the PUT anyway).
- }
-
+function sharedUiChanges(
+ files: Record,
+ remote: Record,
+ keepDeleted?: boolean,
+): SharedUiChange[] {
// Use Object.hasOwn, not `in`: a file named after an Object.prototype member
// (e.g. ui/toString) would otherwise register as always-present and be
// misdiffed.
@@ -69,34 +65,97 @@ export async function diffSharedUi(workspace: string): Promise
changes.push({ type: "edited", path: p, before: remote[rel], after: content });
}
}
- for (const rel of Object.keys(remote)) {
- if (!Object.hasOwn(files, rel)) {
- changes.push({ type: "deleted", path: `${SHARED_UI_DIR}/${rel}` });
+ if (!keepDeleted) {
+ for (const rel of Object.keys(remote)) {
+ if (!Object.hasOwn(files, rel)) {
+ changes.push({ type: "deleted", path: `${SHARED_UI_DIR}/${rel}` });
+ }
}
}
return changes;
}
+/**
+ * Diff the local /ui/ folder against the workspace's shared UI store in
+ * the push direction (local -> remote), returning entries whose `path` is
+ * prefixed with `ui/`. This is the same comparison pushSharedUi applies, so the
+ * dry-run preview and the real push never diverge.
+ *
+ * Mirrors pushSharedUi's no-ops: with no local ui/ folder, or under
+ * `keepDeleted` with an unreadable store, the apply does nothing, so the
+ * preview must be empty (even when the remote store is non-empty) to avoid
+ * phantom diffs the apply won't perform.
+ */
+export async function diffSharedUi(
+ workspace: string,
+ keepDeleted?: boolean,
+): Promise {
+ const localDir = path.join(process.cwd(), SHARED_UI_DIR);
+ if (!fs.existsSync(localDir)) {
+ return [];
+ }
+ const files = await readDirRecursive(localDir);
+ const remote = await fetchSharedUi(workspace);
+ if (remote === undefined && keepDeleted) {
+ return [];
+ }
+ // If endpoint missing or unauthorized, treat remote as empty (the push
+ // would attempt the PUT anyway).
+ return sharedUiChanges(files, remote ?? {}, keepDeleted);
+}
+
/**
* Push the local /ui/ folder to the workspace's shared UI store.
* Returns true if a push was performed, false if the folder is missing or
* already matches the remote store. Note an empty-but-existing folder still
- * pushes an empty map (clearing the remote store) if the remote is non-empty.
+ * pushes an empty map (clearing the remote store) if the remote is non-empty —
+ * unless `keepDeleted`, which folds remote-only files back into the map.
*/
-export async function pushSharedUi(workspace: string): Promise {
+export async function pushSharedUi(
+ workspace: string,
+ keepDeleted?: boolean,
+): Promise {
const localDir = path.join(process.cwd(), SHARED_UI_DIR);
if (!fs.existsSync(localDir)) {
return false;
}
- // Skip if no change — reuse diffSharedUi so preview and push never diverge.
- const diff = await diffSharedUi(workspace);
- if (diff.length === 0) {
+ const files = await readDirRecursive(localDir);
+ const remote = await fetchSharedUi(workspace);
+ // The store is written whole, so a remote-only file is pruned by omission
+ // alone. An unreadable store leaves no way to carry those files over, so
+ // the shared UI is left untouched rather than cleared.
+ if (remote === undefined && keepDeleted) {
+ log.warn(
+ colors.yellow(
+ "Could not read the shared UI folder from the remote; skipping its push so --keep-deleted does not clear it.",
+ ),
+ );
+ return false;
+ }
+
+ // Same comparison as diffSharedUi, off the same two maps, so preview and
+ // push never diverge.
+ if (sharedUiChanges(files, remote ?? {}, keepDeleted).length === 0) {
log.info(colors.gray("Shared UI folder up to date"));
return false;
}
- const files = await readDirRecursive(localDir);
+ if (keepDeleted) {
+ for (const [rel, content] of Object.entries(remote!)) {
+ // defineProperty, not assignment: `files.__proto__ = "..."` runs the
+ // inherited setter and creates no own property, so a remote `ui/__proto__`
+ // would be dropped from the whole-map PUT — i.e. deleted despite the flag.
+ if (!Object.hasOwn(files, rel)) {
+ Object.defineProperty(files, rel, {
+ value: content,
+ writable: true,
+ enumerable: true,
+ configurable: true,
+ });
+ }
+ }
+ }
await wmill.updateSharedUi({
workspace,
requestBody: { files },
@@ -111,9 +170,12 @@ export async function pushSharedUi(workspace: string): Promise {
/**
* Pull the workspace's shared UI store into /ui/.
- * Files removed remotely are also removed locally.
+ * Files removed remotely are also removed locally, unless `keepDeleted`.
*/
-export async function pullSharedUi(workspace: string): Promise {
+export async function pullSharedUi(
+ workspace: string,
+ keepDeleted?: boolean,
+): Promise {
const localDir = path.join(process.cwd(), SHARED_UI_DIR);
let got;
try {
@@ -145,15 +207,17 @@ export async function pullSharedUi(workspace: string): Promise {
}
// Delete locally-orphaned files
- const known = new Set(Object.keys(files));
- const local = await readDirRecursive(localDir);
- for (const rel of Object.keys(local)) {
- if (!known.has(rel)) {
- const full = path.join(localDir, rel);
- try {
- fs.unlinkSync(full);
- } catch {
- // ignore
+ if (!keepDeleted) {
+ const known = new Set(Object.keys(files));
+ const local = await readDirRecursive(localDir);
+ for (const rel of Object.keys(local)) {
+ if (!known.has(rel)) {
+ const full = path.join(localDir, rel);
+ try {
+ fs.unlinkSync(full);
+ } catch {
+ // ignore
+ }
}
}
}
diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts
index b1fca78ed2..cf44d5ec80 100644
--- a/cli/src/commands/sync/sync.ts
+++ b/cli/src/commands/sync/sync.ts
@@ -3174,6 +3174,28 @@ export function untrackedDatatableMigrationDeletions<
);
}
+/**
+ * `--keep-deleted`: strip every deletion from the changeset, in place, so the
+ * sync only adds and updates. A path missing on one side is not on its own
+ * evidence that it should go from the other — a partial clone, a scoped
+ * checkout or an item authored in the UI all read as deletions here.
+ */
+function dropDeletions(changes: Change[], keptOn: "local" | "remote"): void {
+ const deletions = changes.filter((c) => c.name === "deleted");
+ if (deletions.length === 0) return;
+ const kept = changes.filter((c) => c.name !== "deleted");
+ changes.length = 0;
+ changes.push(...kept);
+ log.info(
+ colors.yellow(
+ `--keep-deleted: keeping ${deletions.length} item(s) that exist only ` +
+ (keptOn === "local"
+ ? `on disk instead of deleting them locally`
+ : `on the remote instead of deleting them from the workspace`),
+ ),
+ );
+}
+
interface ChangeTracker {
scripts: string[];
flows: string[];
@@ -3433,6 +3455,7 @@ export async function pull(
repository?: string;
promotion?: string;
branch?: string;
+ keepDeleted?: boolean;
useIndividualBranch?: boolean;
groupByFolder?: boolean;
gitDeployItems?: string;
@@ -3684,6 +3707,10 @@ export async function pull(
await isCaseInsensitiveFilesystem(process.cwd()),
);
+ if (opts.keepDeleted) {
+ dropDeletions(changes, "local");
+ }
+
log.info(
`remote (${workspace.name}) -> local: ${changes.length} changes to apply`,
);
@@ -4117,10 +4144,14 @@ export async function pull(
}
}
- try {
- await pullSharedUi(workspace.workspaceId);
- } catch (e) {
- log.warn(`Failed to pull shared UI folder: ${e}`);
+ // Skipped under --dry-run since pullSharedUi writes to the local ui/ folder.
+ // An empty changeset falls through the return above and reaches here.
+ if (!opts.dryRun) {
+ try {
+ await pullSharedUi(workspace.workspaceId, opts.keepDeleted);
+ } catch (e) {
+ log.warn(`Failed to pull shared UI folder: ${e}`);
+ }
}
// Datatable migrations are part of the workspace export now, so they flow
@@ -4425,12 +4456,19 @@ function removeSuffix(str: string, suffix: string) {
}
// Shown after a `wmill sync push --dry-run` preview that has changes. `sync push`
-// deploys to the remote workspace and is destructive (it overwrites and prunes
-// remote items that differ from or are absent locally), so the preview reminds
-// the caller — especially an AI agent that ran the dry-run to inspect changes —
-// to get explicit user confirmation before applying it for real.
-const SYNC_PUSH_DESTRUCTIVE_WARNING =
- "`wmill sync push` is destructive: applying it deploys these changes to the remote workspace and overwrites or deletes remote items that differ from or are absent locally — this is not automatically reversible. If you are an AI agent, do NOT run `wmill sync push` (without --dry-run) until the user has explicitly confirmed this deploy, unless your custom instructions explicitly allow bypassing that confirmation.";
+// deploys to the remote workspace and is destructive (it overwrites remote items
+// that differ from local, and prunes those absent locally unless --keep-deleted),
+// so the preview reminds the caller — especially an AI agent that ran the dry-run
+// to inspect changes — to get explicit user confirmation before applying it for real.
+function syncPushDestructiveWarning(keepDeleted?: boolean): string {
+ return (
+ "`wmill sync push` is destructive: applying it deploys these changes to the remote workspace and overwrites " +
+ (keepDeleted
+ ? "remote items that differ from local"
+ : "or deletes remote items that differ from or are absent locally") +
+ " — this is not automatically reversible. If you are an AI agent, do NOT run `wmill sync push` (without --dry-run) until the user has explicitly confirmed this deploy, unless your custom instructions explicitly allow bypassing that confirmation."
+ );
+}
// A script pushed without a local lock queues a server-side dependency job; if
// that job fails the script deploys broken (no lock/assets) with no CLI signal.
@@ -4490,6 +4528,7 @@ export async function push(
SyncOptions & {
repository?: string;
branch?: string;
+ keepDeleted?: boolean;
acceptOverridingPermissionedAsWithSelf?: boolean;
},
) {
@@ -4791,6 +4830,13 @@ export async function push(
);
}
+ // After the shared-lock pass, which reads a shared lockfile's deletion as the
+ // signal that this checkout is not deduplicated — an advisory about the local
+ // tree that holds whether or not remote items are being kept.
+ if (opts.keepDeleted) {
+ dropDeletions(changes, "remote");
+ }
+
const autoRegenerate = !!(opts as any).autoMetadata;
const staleScripts: string[] = [];
const staleFlows: string[] = [];
@@ -5103,7 +5149,10 @@ export async function push(
// unchanged (pushSharedUi still runs) and the summary count includes ui/.
if (opts.dryRun) {
try {
- for (const c of await diffSharedUi(workspace.workspaceId)) {
+ for (const c of await diffSharedUi(
+ workspace.workspaceId,
+ opts.keepDeleted,
+ )) {
if (c.type === "added") {
changes.push({ name: "added", path: c.path, content: "" });
} else if (c.type === "deleted") {
@@ -5275,7 +5324,9 @@ export async function push(
: {}),
})),
total: changes.length,
- ...(changes.length > 0 ? { warning: SYNC_PUSH_DESTRUCTIVE_WARNING } : {}),
+ ...(changes.length > 0
+ ? { warning: syncPushDestructiveWarning(opts.keepDeleted) }
+ : {}),
};
console.log(JSON.stringify(result, null, 2));
return;
@@ -5339,7 +5390,9 @@ export async function push(
if (opts.dryRun) {
log.info(colors.gray(`Dry run complete.`));
- log.warn(colors.yellow(`\n⚠ ${SYNC_PUSH_DESTRUCTIVE_WARNING}`));
+ log.warn(
+ colors.yellow(`\n⚠ ${syncPushDestructiveWarning(opts.keepDeleted)}`),
+ );
return;
}
@@ -6313,7 +6366,7 @@ export async function push(
}
}
try {
- await pushSharedUi(workspace.workspaceId);
+ await pushSharedUi(workspace.workspaceId, opts.keepDeleted);
} catch (e) {
log.warn(`Failed to push shared UI folder: ${e}`);
}
@@ -6415,7 +6468,10 @@ export async function push(
let sharedUiPushed = false;
if (!opts.dryRun) {
try {
- sharedUiPushed = await pushSharedUi(workspace.workspaceId);
+ sharedUiPushed = await pushSharedUi(
+ workspace.workspaceId,
+ opts.keepDeleted,
+ );
} catch (e) {
log.warn(`Failed to push shared UI folder: ${e}`);
}
@@ -6480,6 +6536,10 @@ const command = new Command()
.option("--include-groups", "Include syncing groups")
.option("--include-settings", "Include syncing workspace settings")
.option("--include-key", "Include workspace encryption key")
+ .option(
+ "--keep-deleted",
+ "Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates.",
+ )
.option("--skip-branch-validation", "Skip git branch validation and prompts")
.option("--json-output", "Output results in JSON format")
.option(
@@ -6543,6 +6603,10 @@ const command = new Command()
"--skip-reencrypt-on-key-change",
"When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt.",
)
+ .option(
+ "--keep-deleted",
+ "Do not delete remote items that no longer exist locally. Only adds and updates.",
+ )
.option("--skip-branch-validation", "Skip git branch validation and prompts")
.option("--json-output", "Output results in JSON format")
.option(
diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts
index ebd6360875..62027ab2ef 100644
--- a/cli/src/guidance/skills.gen.ts
+++ b/cli/src/guidance/skills.gen.ts
@@ -7519,6 +7519,7 @@ sync local with a remote workspaces or the opposite (push or pull)
- \`--include-groups\` - Include syncing groups
- \`--include-settings\` - Include syncing workspace settings
- \`--include-key\` - Include workspace encryption key
+ - \`--keep-deleted\` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates.
- \`--skip-branch-validation\` - Skip git branch validation and prompts
- \`--json-output\` - Output results in JSON format
- \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes
@@ -7550,6 +7551,7 @@ sync local with a remote workspaces or the opposite (push or pull)
- \`--include-settings\` - Include syncing workspace settings
- \`--include-key\` - Include workspace encryption key
- \`--skip-reencrypt-on-key-change\` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt.
+ - \`--keep-deleted\` - Do not delete remote items that no longer exist locally. Only adds and updates.
- \`--skip-branch-validation\` - Skip git branch validation and prompts
- \`--json-output\` - Output results in JSON format
- \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)
diff --git a/cli/test/shared_ui_diff_unit.test.ts b/cli/test/shared_ui_diff_unit.test.ts
index f2fd69210e..052fa58a8e 100644
--- a/cli/test/shared_ui_diff_unit.test.ts
+++ b/cli/test/shared_ui_diff_unit.test.ts
@@ -11,12 +11,25 @@ import * as os from "node:os";
import * as path from "node:path";
let remoteFiles: Record = {};
+let remoteUnreadable = false;
+let pushedFiles: Record | undefined;
mock.module("../gen/services.gen.ts", () => ({
- getSharedUi: async (_args: { workspace: string }) => ({ files: remoteFiles }),
+ getSharedUi: async (_args: { workspace: string }) => {
+ if (remoteUnreadable) throw new Error("shared UI store unreadable");
+ return { files: remoteFiles };
+ },
+ updateSharedUi: async (args: {
+ workspace: string;
+ requestBody: { files: Record };
+ }) => {
+ pushedFiles = args.requestBody.files;
+ },
}));
-const { diffSharedUi } = await import("../src/commands/shared_ui.ts");
+const { diffSharedUi, pushSharedUi } = await import(
+ "../src/commands/shared_ui.ts"
+);
describe("diffSharedUi", () => {
const ws = "test-workspace";
@@ -25,6 +38,8 @@ describe("diffSharedUi", () => {
beforeEach(() => {
remoteFiles = {};
+ remoteUnreadable = false;
+ pushedFiles = undefined;
prevCwd = process.cwd();
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-shared-ui-"));
process.chdir(tmpDir);
@@ -89,4 +104,46 @@ describe("diffSharedUi", () => {
const changes = await diffSharedUi(ws);
expect(changes).toEqual([]);
});
+
+ test("emits nothing under keepDeleted when the remote store is unreadable", async () => {
+ // pushSharedUi skips the push rather than clearing a store it can't read,
+ // so the preview must show that same nothing.
+ remoteUnreadable = true;
+ writeUi("theme.json", "{}");
+ expect(await diffSharedUi(ws, true)).toEqual([]);
+ });
+});
+
+describe("pushSharedUi with keepDeleted", () => {
+ const ws = "test-workspace";
+ let tmpDir: string;
+ let prevCwd: string;
+
+ beforeEach(() => {
+ remoteFiles = {};
+ remoteUnreadable = false;
+ pushedFiles = undefined;
+ prevCwd = process.cwd();
+ tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-shared-ui-push-"));
+ process.chdir(tmpDir);
+ });
+
+ afterEach(() => {
+ process.chdir(prevCwd);
+ fs.rmSync(tmpDir, { recursive: true, force: true });
+ });
+
+ test("carries remote-only files, including ui/__proto__, into the pushed map", async () => {
+ // JSON.parse, not a literal: `{__proto__: …}` sets the prototype instead of
+ // creating the own property the API response really has.
+ remoteFiles = JSON.parse('{"__proto__":"keep me","extra.json":"1"}');
+ fs.mkdirSync(path.join(tmpDir, "ui"), { recursive: true });
+ fs.writeFileSync(path.join(tmpDir, "ui", "theme.json"), "{}", "utf-8");
+
+ expect(await pushSharedUi(ws, true)).toBe(true);
+ // The store is written whole, so anything missing here is deleted.
+ expect(pushedFiles!["extra.json"]).toEqual("1");
+ expect(Object.getOwnPropertyDescriptor(pushedFiles!, "__proto__")?.value)
+ .toEqual("keep me");
+ });
});
diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts
index 343ed49ef7..24c4a282a3 100644
--- a/cli/test/sync_pull_push.test.ts
+++ b/cli/test/sync_pull_push.test.ts
@@ -2774,3 +2774,102 @@ kind: script
});
});
});
+
+describe("keep deleted", () => {
+ test("Integration: --keep-deleted keeps items absent from the other side", async () => {
+ await withTestBackend(async (backend, tempDir) => {
+ const uniqueId = Date.now();
+ const scriptPath = `f/test/keep_deleted_${uniqueId}`;
+
+ const resp = await backend.apiRequest!(
+ `/api/w/${backend.workspace}/scripts/create`,
+ {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ path: scriptPath,
+ content: 'export async function main() { return "keep me"; }',
+ language: "bun",
+ summary: "Kept by --keep-deleted",
+ schema: {
+ $schema: "https://json-schema.org/draft/2020-12/schema",
+ type: "object",
+ properties: {},
+ required: [],
+ },
+ }),
+ }
+ );
+ expect(resp.status).toBeLessThan(300);
+ await resp.text();
+
+ await writeWmillYaml(tempDir);
+ expect(
+ (await backend.runCLICommand(["sync", "pull", "--yes"], tempDir)).code
+ ).toEqual(0);
+
+ const contentFile = `${scriptPath}.ts`;
+ const metadataFile = `${scriptPath}.script.yaml`;
+ expect(await listFilesRecursive(tempDir)).toContain(contentFile);
+
+ // Push direction: the remote script survives losing its local files.
+ await rm(join(tempDir, contentFile));
+ await rm(join(tempDir, metadataFile));
+ expect(
+ (
+ await backend.runCLICommand(
+ ["sync", "push", "--yes", "--keep-deleted"],
+ tempDir
+ )
+ ).code
+ ).toEqual(0);
+ // A push deletion archives the script rather than removing the row, so
+ // `archived` — not the status code — is what says it survived.
+ const remote = await backend.apiRequest!(
+ `/api/w/${backend.workspace}/scripts/get/p/${scriptPath}`
+ );
+ expect(remote.status).toEqual(200);
+ expect((await remote.json()).archived).not.toEqual(true);
+
+ // Pull direction: a file with no remote counterpart survives the pull.
+ const localOnly = `f/test/local_only_${uniqueId}.ts`;
+ await writeFile(
+ join(tempDir, localOnly),
+ 'export async function main() { return "local only"; }',
+ "utf-8"
+ );
+ expect(
+ (
+ await backend.runCLICommand(
+ ["sync", "pull", "--yes", "--keep-deleted"],
+ tempDir
+ )
+ ).code
+ ).toEqual(0);
+ const afterPull = await listFilesRecursive(tempDir);
+ expect(afterPull).toContain(localOnly);
+ // Adds still apply: the script deleted above is written back.
+ expect(afterPull).toContain(contentFile);
+
+ // An empty changeset falls past the dry-run return, on to the shared-UI
+ // step — which writes to disk, so a dry run must skip it.
+ // Including the metadata and lock the pull's auto-fill generated for it.
+ for (const ext of [".ts", ".script.yaml", ".script.lock"]) {
+ await rm(join(tempDir, `f/test/local_only_${uniqueId}${ext}`), {
+ force: true,
+ });
+ }
+ await mkdir(join(tempDir, "ui"), { recursive: true });
+ await writeFile(join(tempDir, "ui", "custom.css"), "body{}", "utf-8");
+ const dryRun = await backend.runCLICommand(
+ ["sync", "pull", "--dry-run"],
+ tempDir
+ );
+ expect(dryRun.code).toEqual(0);
+ // Guards against a vacuous pass: a non-empty changeset would return at
+ // the dry-run check above and never reach the shared-UI step.
+ expect(dryRun.stdout + dryRun.stderr).toContain("0 changes to apply");
+ expect(await listFilesRecursive(tempDir)).toContain("ui/custom.css");
+ });
+ });
+});
diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md
index 8029ae48c7..e51c8371dc 100644
--- a/system_prompts/auto-generated/cli/cli-commands.md
+++ b/system_prompts/auto-generated/cli/cli-commands.md
@@ -605,6 +605,7 @@ sync local with a remote workspaces or the opposite (push or pull)
- `--include-groups` - Include syncing groups
- `--include-settings` - Include syncing workspace settings
- `--include-key` - Include workspace encryption key
+ - `--keep-deleted` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates.
- `--skip-branch-validation` - Skip git branch validation and prompts
- `--json-output` - Output results in JSON format
- `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes
@@ -636,6 +637,7 @@ sync local with a remote workspaces or the opposite (push or pull)
- `--include-settings` - Include syncing workspace settings
- `--include-key` - Include workspace encryption key
- `--skip-reencrypt-on-key-change` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt.
+ - `--keep-deleted` - Do not delete remote items that no longer exist locally. Only adds and updates.
- `--skip-branch-validation` - Skip git branch validation and prompts
- `--json-output` - Output results in JSON format
- `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)
diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts
index 2c3b585c20..918b88cb21 100644
--- a/system_prompts/auto-generated/prompts.ts
+++ b/system_prompts/auto-generated/prompts.ts
@@ -3671,6 +3671,7 @@ sync local with a remote workspaces or the opposite (push or pull)
- \`--include-groups\` - Include syncing groups
- \`--include-settings\` - Include syncing workspace settings
- \`--include-key\` - Include workspace encryption key
+ - \`--keep-deleted\` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates.
- \`--skip-branch-validation\` - Skip git branch validation and prompts
- \`--json-output\` - Output results in JSON format
- \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes
@@ -3702,6 +3703,7 @@ sync local with a remote workspaces or the opposite (push or pull)
- \`--include-settings\` - Include syncing workspace settings
- \`--include-key\` - Include workspace encryption key
- \`--skip-reencrypt-on-key-change\` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt.
+ - \`--keep-deleted\` - Do not delete remote items that no longer exist locally. Only adds and updates.
- \`--skip-branch-validation\` - Skip git branch validation and prompts
- \`--json-output\` - Output results in JSON format
- \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)
diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md
index ebd0d7f4da..1f3c54ea26 100644
--- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md
+++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md
@@ -610,6 +610,7 @@ sync local with a remote workspaces or the opposite (push or pull)
- `--include-groups` - Include syncing groups
- `--include-settings` - Include syncing workspace settings
- `--include-key` - Include workspace encryption key
+ - `--keep-deleted` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates.
- `--skip-branch-validation` - Skip git branch validation and prompts
- `--json-output` - Output results in JSON format
- `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes
@@ -641,6 +642,7 @@ sync local with a remote workspaces or the opposite (push or pull)
- `--include-settings` - Include syncing workspace settings
- `--include-key` - Include workspace encryption key
- `--skip-reencrypt-on-key-change` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt.
+ - `--keep-deleted` - Do not delete remote items that no longer exist locally. Only adds and updates.
- `--skip-branch-validation` - Skip git branch validation and prompts
- `--json-output` - Output results in JSON format
- `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)
From b57e231c2bf5e5fe007f0aa7b958a51e32b47141 Mon Sep 17 00:00:00 2001
From: Guilhem
Date: Mon, 31 Aug 2026 16:13:32 +0200
Subject: [PATCH 022/310] fix: keep raw-app editor selection consistent across
sidebar and tabs (#10885)
* fix: route raw-app editor selection through one switch function
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H
* fix: stop announcing folders as selected from the file tree
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H
* fix: carry the selection through a folder rename
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H
* fix: keep the generated wmill.ts tab out of stale-tab cleanup
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H
* refactor: test document existence through one predicate
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H
* refactor: route the history replay through the same predicate
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H
* fix: clear the selection in the same tick a runnable is deleted
Deleting the selected runnable dropped it from `runnables` and left the
editor to notice via the stale-tab effect, one frame later. In that window
the pane rendered "No runnable at id ".
The sidebar list now reports the delete instead of mutating `runnables`
itself; the editor deletes and closes the tab together, so the selection
moves through `select` synchronously. The stale-tab effect stays as the
backstop for deletes that come from elsewhere.
Also retitle the two sidebar create buttons and rename the FileExplorer
exports behind them: both have always anchored on the selected file's
parent folder, never the root.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H
* refactor: require the runnable delete callback
Optional, the row's Delete button renders and does nothing. There is one
caller and it always supplies it, so the compiler can hold that.
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_018LPVrfeXbjznotqG7JdF4H
---------
Co-authored-by: Claude Opus 5 (1M context)
---
.../src/lib/components/FileExplorer.svelte | 73 +++----
.../components/raw_apps/FileTreeNode.svelte | 34 ++-
.../components/raw_apps/RawAppEditor.svelte | 195 +++++++++++-------
.../RawAppInlineScriptPanelList.svelte | 20 +-
.../components/raw_apps/RawAppSidebar.svelte | 41 ++--
frontend/src/lib/components/raw_apps/utils.ts | 3 +
6 files changed, 204 insertions(+), 162 deletions(-)
diff --git a/frontend/src/lib/components/FileExplorer.svelte b/frontend/src/lib/components/FileExplorer.svelte
index a92972f3e9..e031c21337 100644
--- a/frontend/src/lib/components/FileExplorer.svelte
+++ b/frontend/src/lib/components/FileExplorer.svelte
@@ -8,9 +8,11 @@
interface Props {
/** File path → content map. Keys use / prefix (e.g. /index.html). */
files: Record
- /** Currently selected path (/-prefixed). Read-only; changes via onSelectPath callback. */
+ /** Currently selected file (/-prefixed). Read-only; changes via onSelectPath callback. */
selectedPath?: string | undefined
- /** Called when user clicks a path (file or folder). */
+ /** Called when the user clicks a file. Folders aren't selectable — clicking
+ * one only expands it — so the only non-file paths this reports are the root
+ * row under `showRoot`, and '' when the last file is deleted. */
onSelectPath?: (path: string) => void
/** Extra tree nodes appended after the main tree (e.g. read-only wmill.ts). */
extraNodes?: TreeNode[]
@@ -80,6 +82,12 @@
onSelectPath?.(path)
}
+ function parentFolderOfSelection(): string {
+ if (!selectedPath || selectedPath === '/') return '/'
+ const pathParts = selectedPath.split('/').filter(Boolean)
+ return pathParts.length > 1 ? '/' + pathParts.slice(0, -1).join('/') + '/' : '/'
+ }
+
function handleAddFile(folderPath: string) {
const normalizedFolder = folderPath.endsWith('/') ? folderPath : folderPath + '/'
const basePath = normalizedFolder + 'newfile.txt'
@@ -88,20 +96,10 @@
pathToEdit = newPath
}
- export function handleAddRootFile() {
- let basePath: string
- if (selectedPath && selectedPath !== '/') {
- if (selectedPath.endsWith('/')) {
- basePath = selectedPath + 'newfile.txt'
- } else {
- const pathParts = selectedPath.split('/').filter(Boolean)
- const parentPath =
- pathParts.length > 1 ? '/' + pathParts.slice(0, -1).join('/') + '/' : '/'
- basePath = parentPath + 'newfile.txt'
- }
- } else {
- basePath = '/newfile.txt'
- }
+ // New entries land beside the selected file; with nothing selected, at the
+ // root. To create inside another folder, use that folder row's own menu.
+ export function handleAddFileBesideSelection() {
+ const basePath = parentFolderOfSelection() + 'newfile.txt'
const newPath = getUniquePath(basePath)
pendingNewFilePath = newPath
pathToEdit = newPath
@@ -115,20 +113,8 @@
pathToEdit = newPath
}
- export function handleAddRootFolder() {
- let basePath: string
- if (selectedPath && selectedPath !== '/') {
- if (selectedPath.endsWith('/')) {
- basePath = selectedPath + 'newfolder/'
- } else {
- const pathParts = selectedPath.split('/').filter(Boolean)
- const parentPath =
- pathParts.length > 1 ? '/' + pathParts.slice(0, -1).join('/') + '/' : '/'
- basePath = parentPath + 'newfolder/'
- }
- } else {
- basePath = '/newfolder/'
- }
+ export function handleAddFolderBesideSelection() {
+ const basePath = parentFolderOfSelection() + 'newfolder/'
const newPath = getUniquePath(basePath)
pendingNewFilePath = newPath
pathToEdit = newPath
@@ -168,9 +154,7 @@
}
// Also rename in emptyFolders
emptyFolders = emptyFolders.map((f) =>
- f === oldPath || f.startsWith(oldPath)
- ? newPath + f.substring(oldPath.length)
- : f
+ f === oldPath || f.startsWith(oldPath) ? newPath + f.substring(oldPath.length) : f
)
}
} else {
@@ -187,7 +171,13 @@
files = nfiles
pathToEdit = undefined
- onSelectPath?.(newPath)
+ if (!isFolder) {
+ onSelectPath?.(newPath)
+ } else if (selectedPath?.startsWith(oldPath)) {
+ // A folder isn't selectable, but the selected file moved with it — follow
+ // it to its new path, or the caller keeps editing a key that's now gone.
+ onSelectPath?.(newPath + selectedPath.slice(oldPath.length))
+ }
}
function handleDelete(path: string) {
@@ -208,12 +198,8 @@
files = nfiles
if (selectedPath === path || (isFolder && selectedPath?.startsWith(path))) {
- const remaining = Object.keys(nfiles)
- if (remaining.length > 0) {
- onSelectPath?.(remaining[0])
- } else {
- onSelectPath?.(showRoot ? '/' : '')
- }
+ const remainingFile = Object.keys(nfiles).find((key) => !key.endsWith('/'))
+ onSelectPath?.(remainingFile ?? (showRoot ? '/' : ''))
}
}
@@ -223,7 +209,7 @@
Files
diff --git a/frontend/src/lib/components/raw_apps/FileTreeNode.svelte b/frontend/src/lib/components/raw_apps/FileTreeNode.svelte
index 3d428b9aaf..0e02701347 100644
--- a/frontend/src/lib/components/raw_apps/FileTreeNode.svelte
+++ b/frontend/src/lib/components/raw_apps/FileTreeNode.svelte
@@ -1,5 +1,15 @@
@@ -125,16 +127,8 @@
{runnable}
isSelected={selectedRunnable === id}
isEditing={editingId === id}
- onSelect={() => {
- selectedRunnable = id
- onSelect?.(id)
- }}
- onDelete={() => {
- delete runnables[id]
- if (selectedRunnable === id) {
- selectedRunnable = undefined
- }
- }}
+ onSelect={() => onSelect?.(id)}
+ onDelete={() => onDelete(id)}
onRename={(newId) => renameRunnable(id, newId)}
onRequestEdit={() => (editingId = id)}
onCancelEdit={() => (editingId = undefined)}
diff --git a/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte b/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte
index b8d6e33c87..fbe02e1dc9 100644
--- a/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte
+++ b/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte
@@ -3,6 +3,7 @@
SUBTLE_PANEL_TITLE
} from '../apps/editor/settingsPanel/common/PanelSection.svelte'
import type { Runnable } from '../apps/inputType'
+ import { WMILL_TS_PATH } from './utils'
import RawAppInlineScriptPanelList from './RawAppInlineScriptPanelList.svelte'
import FileExplorer from '../FileExplorer.svelte'
import { Plus, File, Folder, Camera } from 'lucide-svelte'
@@ -18,10 +19,14 @@
interface Props {
runnables: Record
+ /** Read-only; the editor switches selection through `onSelectRunnable`. */
selectedRunnable: string | undefined
files: Record
modules?: Modules
- onSelectFile?: (path: string) => void
+ onSelectRunnable?: (key: string) => void
+ onDeleteRunnable: (key: string) => void
+ onSelectPath?: (path: string) => void
+ /** Read-only; the editor switches selection through `onSelectPath`. */
selectedDocument: string | undefined
historyManager?: RawAppHistoryManager
historySelectedId?: number | undefined
@@ -39,11 +44,13 @@
let {
runnables,
- selectedRunnable = $bindable(),
+ selectedRunnable,
files = $bindable({}),
modules,
- onSelectFile,
- selectedDocument = $bindable(),
+ onSelectRunnable,
+ onDeleteRunnable,
+ onSelectPath,
+ selectedDocument,
historyManager,
historySelectedId,
onHistorySelect,
@@ -79,13 +86,6 @@
}
let fileExplorer: FileExplorer | undefined = $state()
-
- function handleSelectPath(path: string) {
- selectedDocument = path
- if (!path.endsWith('/')) {
- onSelectFile?.(path)
- }
- }
@@ -142,11 +142,10 @@
{
- selectedDocument = undefined
- }}
+ onSelect={onSelectRunnable}
+ onDelete={onDeleteRunnable}
/>
diff --git a/frontend/src/lib/components/raw_apps/utils.ts b/frontend/src/lib/components/raw_apps/utils.ts
index e353c6f825..244a9ba1cc 100644
--- a/frontend/src/lib/components/raw_apps/utils.ts
+++ b/frontend/src/lib/components/raw_apps/utils.ts
@@ -233,6 +233,9 @@ function hiddenRunnableToTsType(runnable: Runnable) {
}
}
+/** Shown in the file tree, generated by `genWmillTs` — never a key of `files`. */
+export const WMILL_TS_PATH = '/wmill.ts'
+
export function genWmillTs(runnables: Record) {
return `// THIS FILE IS READ-ONLY
// AND GENERATED AUTOMATICALLY FROM YOUR RUNNABLES
From 831370cdde8e06f4298b17baa1a0041bacdd98c7 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel
Date: Mon, 31 Aug 2026 18:01:28 +0200
Subject: [PATCH 023/310] fix: harden the service log indexer's recovery and
read paths (#10904)
* [ee] fix: an unreadable ingest cursor should not stop the server booting
Three follow-ups to #10894, all in the service log indexer: a corrupt cursor no
longer takes the server down at boot, the queue's writes are covered against a
real database rather than by hand, and a read skips the dedupe when the partition
it reads holds a single object.
* [ee] test: place the queue's rows relative to the clock the statement reads
Also drops the two `.sqlx` entries the query extraction orphaned: sqlx keys on the
literal including its indentation, so moving a query into a function leaves the
old copy behind.
* [ee] test: make the pair-exactness and rebuild-dedupe tests actually bite
* [ee] docs: state the cursor and dedupe rules without their history
* chore: update ee-repo-ref to 90a368362896ebcc2fcfaaf9510dc9be68c929f7
This commit updates the EE repository reference after PR #761 was merged in windmill-ee-private.
Previous ee-repo-ref: e3423705aa8f2d585bc65474cfd0c4c762ec4ad5
New ee-repo-ref: 90a368362896ebcc2fcfaaf9510dc9be68c929f7
Automated by sync-ee-ref workflow.
---------
Co-authored-by: windmill-internal-app[bot]
---
...f927ac4886755fd17ec755037763aa92f6c0e.json | 22 +++++++++++++++++++
...33fa0fd05b067454f0d4686d8b155fe562ffe.json | 15 +++++++++++++
...cdc85389c0629847d8fabb2ad0aa043957b2f.json | 22 -------------------
...0c06924719189bec8e5c19866db8d0b87df5e.json | 15 -------------
backend/ee-repo-ref.txt | 2 +-
5 files changed, 38 insertions(+), 38 deletions(-)
create mode 100644 backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json
create mode 100644 backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json
delete mode 100644 backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json
delete mode 100644 backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json
diff --git a/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json b/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json
new file mode 100644
index 0000000000..98e08b8167
--- /dev/null
+++ b/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "WITH moved AS (\n UPDATE log_file SET indexed_at = CASE\n WHEN log_ts > NOW() - make_interval(secs => $1) THEN NULL\n ELSE now() END\n WHERE indexed_at = 'epoch' RETURNING 1)\n SELECT count(*) AS \"n!\" FROM moved",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "n!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Float8"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e"
+}
diff --git a/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json b/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json
new file mode 100644
index 0000000000..76456ea8fb
--- /dev/null
+++ b/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::timestamp[]) AS c(hostname, log_ts)\n WHERE log_file.hostname = c.hostname AND log_file.log_ts = c.log_ts",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "TimestampArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe"
+}
diff --git a/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json b/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json
deleted file mode 100644
index 96745f9485..0000000000
--- a/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "WITH moved AS (\n UPDATE log_file SET indexed_at = CASE\n WHEN log_ts > NOW() - make_interval(secs => $1) THEN NULL\n ELSE now() END\n WHERE indexed_at = 'epoch' RETURNING 1)\n SELECT count(*) AS \"n!\" FROM moved",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "n!",
- "type_info": "Int8"
- }
- ],
- "parameters": {
- "Left": [
- "Float8"
- ]
- },
- "nullable": [
- null
- ]
- },
- "hash": "8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f"
-}
diff --git a/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json b/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json
deleted file mode 100644
index 6739d80033..0000000000
--- a/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::timestamp[]) AS c(hostname, log_ts)\n WHERE log_file.hostname = c.hostname AND log_file.log_ts = c.log_ts",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "TextArray",
- "TimestampArray"
- ]
- },
- "nullable": []
- },
- "hash": "947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e"
-}
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index 693921efc6..5f2f41186e 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-bc0c7051585194474078b6c1941a3fb73893d9e5
+90a368362896ebcc2fcfaaf9510dc9be68c929f7
From 0c2eb0ae3d18f49c21370131d15011e8dd103746 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel
Date: Mon, 31 Aug 2026 18:08:19 +0200
Subject: [PATCH 024/310] perf: add service log documents to the index one
batch at a time (#10906)
* perf: add service log documents to the index one batch at a time
Co-Authored-By: Claude Opus 5 (1M context)
Claude-Session: https://claude.ai/code/session_014KnE8my2okxQGCx47cWMjf
* chore: update ee-repo-ref to df60763d1f243b0048dfc3fe700bc026b257bea8
This commit updates the EE repository reference after PR #762 was merged in windmill-ee-private.
Previous ee-repo-ref: f9a0b98080eecdc2885720e0f8506933a0675bb5
New ee-repo-ref: df60763d1f243b0048dfc3fe700bc026b257bea8
Automated by sync-ee-ref workflow.
---------
Co-authored-by: Claude Opus 5 (1M context)
Co-authored-by: windmill-internal-app[bot]
---
backend/ee-repo-ref.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index 5f2f41186e..51788b0e1d 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-90a368362896ebcc2fcfaaf9510dc9be68c929f7
+df60763d1f243b0048dfc3fe700bc026b257bea8
From b998267c91b9dcf02787768f6205cc5aeda494fb Mon Sep 17 00:00:00 2001
From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com>
Date: Mon, 31 Aug 2026 18:34:25 +0200
Subject: [PATCH 025/310] fix: show a loading indicator while the initial data
table migration is generated (#10900)
Claude-Session: https://claude.ai/code/session_01R3YQT3BShZ3ivp25yQ6Smd
Co-authored-by: Claude Opus 5 (1M context)
---
.../workspaceSettings/DataTableMigrationsButton.svelte | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte b/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte
index eac0153993..fc9bddcf4e 100644
--- a/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte
+++ b/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte
@@ -52,6 +52,8 @@
let loadError = $state(undefined)
let loading = $state(false)
let busy = $state(false)
+ // pg_dump can take a while, so the snapshot gets its own flag to spin its button.
+ let generatingInitial = $state(false)
// Only workspace admins and super admins can opt a data table in or out.
const canManage = $derived(!!$userStore?.is_admin || !!$superadmin)
@@ -280,6 +282,7 @@
})
if (!confirmed) return
busy = true
+ generatingInitial = true
try {
await WorkspaceService.generateInitialDatatableMigration({
workspace,
@@ -291,6 +294,7 @@
sendUserToast(`Failed to generate initial migration: ${e?.body ?? e?.message ?? e}`, true)
} finally {
busy = false
+ generatingInitial = false
}
}
@@ -493,12 +497,13 @@
{#if migrations.length === 0}
- No migrations yet
+ {generatingInitial ? 'Snapshotting current schema…' : 'No migrations yet'}
-
+
+
Cases{workingCases.length}
@@ -369,13 +372,13 @@
Add a case
-
+
(casesEditing = v)}
/>
@@ -397,7 +400,7 @@
unifiedSize="md"
variant="accent"
loading={saving}
- disabled={writing || !path || !!pathError || (nothingToSave && !casesEditing)}
+ disabled={writing || !path || !!pathError || nothingToSave}
onclick={saveDataset}
>
Save
@@ -408,7 +411,8 @@
variant="accent"
startIcon={{ icon: Plus }}
loading={creating}
- disabled={creating || !path || !!pathError}
+ disabled={creating || !path || !!pathError || noCases}
+ title={noCasesTitle}
onclick={createDataset}
>
Create dataset
diff --git a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte
index d4d1714fd0..63fe366628 100644
--- a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte
+++ b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte
@@ -47,16 +47,6 @@
let hasDraft = $derived(editedConfig !== undefined)
let dataset = $state(undefined)
let hoveringDataset = $state(false)
- /** Set while the dialog stands aside for the dataset drawer, holding the pane's dataset as it
- * was on the way out: coming back onto a different one is the detour's answer, and adopted. */
- let steppedAside = $state<{ dataset: string | undefined } | undefined>(undefined)
-
- /** Hands the screen to the dataset drawer, to be given back when it closes. */
- function stepAside(go: () => void) {
- steppedAside = { dataset: defaultDataset }
- open = false
- go()
- }
/** The agent's versions, for pinning one. Loaded when the dialog opens rather than held: a
* version list goes stale the moment the agent is saved again. */
@@ -76,21 +66,26 @@
}
}
+ /** Whether the dialog was already open on the previous run of the effect below, which reads it
+ * to tell an open from the pane's dataset moving underneath. */
+ let wasOpen = false
+
$effect(() => {
- if (!open) return
+ const isOpen = open
+ const pane = defaultDataset
untrack(() => {
- loadVersions()
- const aside = steppedAside
- steppedAside = undefined
- if (aside) {
- // Back from the drawer: a dataset created or edited there moves the pane onto it, and
- // anything else leaves the field as it was left.
- if (defaultDataset !== aside.dataset) dataset = defaultDataset
+ if (!isOpen) {
+ wasOpen = false
return
}
- // Seeded on every open: the dataset last worked in, and the state of the agent there is
- // most reason to measure.
- dataset = defaultDataset
+ // Followed while the dialog stands, not only read at open: the dataset drawer opens over
+ // this dialog rather than in place of it, so creating, renaming or deleting a dataset
+ // there moves the pane's selection with the dialog still up. Nothing else moves it then.
+ dataset = pane
+ if (wasOpen) return
+ wasOpen = true
+ loadVersions()
+ // The state of the agent there is most reason to measure.
choice = hasDraft ? 'draft' : 'deployed'
})
})
@@ -184,7 +179,7 @@
unifiedSize="sm"
variant="default"
startIcon={{ icon: Plus }}
- onclick={() => stepAside(onNewDataset)}
+ onclick={onNewDataset}
>
New dataset
@@ -215,7 +210,7 @@
title="Edit this dataset"
on:click={() => {
close()
- stepAside(() => onEditDataset(item.value ?? ''))
+ onEditDataset(item.value ?? '')
}}
/>
{/snippet}
@@ -228,7 +223,7 @@
btnClasses="w-full !h-auto !justify-start !rounded-none flex items-center gap-2 px-3 py-2 text-xs !font-normal text-secondary hover:bg-surface-hover"
onClick={() => {
close()
- stepAside(onNewDataset)
+ onNewDataset()
}}
>
@@ -247,7 +242,7 @@
startIcon={{ icon: Pencil }}
iconOnly
title="Edit this dataset"
- on:click={() => stepAside(() => onEditDataset(dataset ?? ''))}
+ on:click={() => onEditDataset(dataset ?? '')}
/>
{/if}
diff --git a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte
index 5d84ef88ce..b544781902 100644
--- a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte
+++ b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte
@@ -7,17 +7,21 @@
import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte'
import TimeAgo from '$lib/components/TimeAgo.svelte'
import { Button } from '$lib/components/common'
- import { Bot, Code2, Loader2, Plus } from 'lucide-svelte'
+ import { Bot, ChevronRight, Code2, Loader2, Plus } from 'lucide-svelte'
+ import { overlayHostActive, topmostSurface } from '$lib/components/common/overlayHost.svelte'
import type { EvalDataset, EvalExperiment, ExperimentScore } from '$lib/gen'
import { datasetSummary, experimentName, formatScore, subjectLabel } from './evalUtils'
let {
experiments,
datasets,
+ caseProgress,
loaded,
+ active = false,
deployedHash = undefined,
currentVersion = undefined,
onOpen,
+ onHighlight,
onEditDataset,
onNew
}: {
@@ -25,17 +29,87 @@
experiments: EvalExperiment[]
/** Whether the list has been read: an empty table is a statement about the agent. */
loaded: boolean
+ /** Whether this list is the page on screen. The keyboard is only answered while it is: the
+ * run page keeps its own rows, and both would otherwise move on one press. */
+ active?: boolean
/** The workspace's datasets, for naming the one a run is of by what it is for. */
datasets: EvalDataset[]
+ /** How many cases each still-running run has finished, keyed by run id. A run the flow has
+ * not been read for yet is at none of them rather than absent: the count is on the row from
+ * the moment it appears, so it never arrives late and shifts the column. */
+ caseProgress: Record
/** What the agent hashes to as deployed, and the version it is on: they resolve a run of
* edits that were later saved, so a run is labelled here as the run picker labels it. */
deployedHash?: string
currentVersion?: number
onOpen: (experiment: EvalExperiment) => void
+ /** The highlighted run, reported up so the surface can act on it — arrowing into the run
+ * page opens the run under the highlight rather than whichever was opened last. */
+ onHighlight?: (id: string | undefined) => void
onEditDataset: (dataset: string) => void
onNew: () => void
} = $props()
+ /** The highlighted run, by id. One state for both the pointer and the keyboard, as a melt menu
+ * does it: hovering a row moves the highlight to it, so the arrows carry on from wherever the
+ * pointer left off instead of running a second, invisible cursor of their own. It says where
+ * the highlight is, not what is chosen — a run is not opened until Enter.
+ *
+ * By id and not by index: the list is newest-first and the poll prepends to it, so an index
+ * would quietly come to mean a different run and Enter would open the wrong one. */
+ let cursorId = $state(undefined)
+ let cursor = $derived(
+ cursorId === undefined ? -1 : experiments.findIndex((e) => e.id === cursorId)
+ )
+ let body: HTMLTableSectionElement | undefined = $state()
+
+ // A window listener answers keys aimed anywhere, so it has to ask two questions the DOM cannot:
+ // is my host the visible one — session preview tabs stay mounted when hidden — and is my surface
+ // still the one on top, rather than under a drawer or a dialog opened since.
+ const hostActive = overlayHostActive()
+ const onTop = topmostSurface()
+ const listening = () => hostActive() && onTop()
+
+ $effect(() => {
+ onHighlight?.(cursorId)
+ })
+
+ // A highlight on a run that has since gone, and the highlight itself when the list is not the
+ // page on screen.
+ $effect(() => {
+ if (!active || (cursorId !== undefined && cursor < 0)) cursorId = undefined
+ })
+
+ function move(by: number) {
+ if (experiments.length === 0) return
+ const from = cursor < 0 ? (by > 0 ? -1 : experiments.length) : cursor
+ const at = Math.max(0, Math.min(experiments.length - 1, from + by))
+ cursorId = experiments[at]?.id
+ // `nearest`, so arrowing through a long list scrolls by a row rather than jumping the table.
+ requestAnimationFrame(() =>
+ body?.querySelectorAll('tr')[at]?.scrollIntoView({ block: 'nearest' })
+ )
+ }
+
+ function onKeydown(event: KeyboardEvent) {
+ if (!active || !listening() || event.metaKey || event.ctrlKey || event.altKey) return
+ const el = event.target as HTMLElement | null
+ if (el?.closest?.('input, textarea, select, [contenteditable="true"], [role="listbox"]')) return
+ if (event.key === 'ArrowDown') {
+ event.preventDefault()
+ move(1)
+ } else if (event.key === 'ArrowUp') {
+ event.preventDefault()
+ move(-1)
+ } else if (event.key === 'Enter' && experiments[cursor]) {
+ // Enter belongs to whatever is focused if that thing does something with it. A highlighted
+ // row is not a reason to swallow the press on `New evaluation` or a row's dataset button.
+ if (el?.closest?.('button, a[href], [role="button"], summary')) return
+ event.preventDefault()
+ onOpen(experiments[cursor])
+ }
+ }
+
/** The one number a column reports: a pass rate where it has a line to pass, the mean where it
* does not. */
function headline(score: ExperimentScore): string | undefined {
@@ -44,13 +118,16 @@
}
+
+
-
+
+
@@ -58,12 +135,20 @@
DatasetCasesScores
- When
+ When
+
-
- {#each experiments as experiment (experiment.id)}
- onOpen(experiment)}>
+
+ {#each experiments as experiment, i (experiment.id)}
+
+ onOpen(experiment)}
+ on:hover={(e) => e.detail && (cursorId = experiment.id)}
+ >
{#if scorer.pass_if != undefined}
diff --git a/frontend/src/lib/components/aiEvals/EvalsPane.svelte b/frontend/src/lib/components/aiEvals/EvalsPane.svelte
index d9e5b16fe4..55792ba8e2 100644
--- a/frontend/src/lib/components/aiEvals/EvalsPane.svelte
+++ b/frontend/src/lib/components/aiEvals/EvalsPane.svelte
@@ -9,9 +9,11 @@
import Label from '$lib/components/Label.svelte'
import Popover from '$lib/components/Popover.svelte'
import { Splitpanes, Pane } from 'svelte-splitpanes'
+ import AnimatedPane from '$lib/components/splitPanes/AnimatedPane.svelte'
import {
type AgentDraft,
AiEvalsService,
+ JobService,
type EvalCase,
type EvalDataset,
type EvalExperiment,
@@ -35,9 +37,11 @@
Code2,
ExternalLink
} from 'lucide-svelte'
+ import PagedContent from '$lib/components/common/modal/PagedContent.svelte'
import EvalDatasetDrawer from './EvalDatasetDrawer.svelte'
import EvalRunsList from './EvalRunsList.svelte'
import EvalRunDialog from './EvalRunDialog.svelte'
+ import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte'
import GfmMarkdown from '$lib/components/GfmMarkdown.svelte'
import {
caseLabel,
@@ -52,6 +56,10 @@
/** A dataset is capped at this many cases, so one page holds the whole set. */
const CASE_PAGE_SIZE = 1000
+ /** The id the run's flow gives the loop over its cases (`CASES_NODE_ID` in `ai_evals/run.rs`).
+ * Looked up by id rather than by position: the flow has a step after the loop too. */
+ const CASES_MODULE_ID = 'cases'
+
let {
agentPath,
opWorkspace = undefined,
@@ -91,12 +99,56 @@
/** What the agent hashes to as deployed: a run of edits carrying it ran what was then saved. */
let deployedHash = $state(undefined)
let running = $state(false)
- let scorers = $derived(dataset?.scorers ?? [])
+ /** The run on screen belongs to a dataset still being read. Until it arrives the rows and the
+ * scorer columns would both be built from the *previous* dataset, so both are held back. */
+ let datasetLoading = $state(false)
+ let scorers = $derived(datasetLoading ? [] : (dataset?.scorers ?? []))
let selectedCaseId = $state(undefined)
let datasetDrawer: EvalDatasetDrawer | undefined = $state()
let runDialogOpen = $state(false)
- let resumeRunDialog = $state(false)
+ /** The run the list has highlighted, so arrowing into the run page opens that one. Without it
+ * the arrow could only fall back to whichever run was opened last, which on a dialog just
+ * opened is none at all. */
+ let highlightedRunId = $state(undefined)
+
+ /** How many cases each still-running run has finished, keyed by run id. Read from the flow
+ * executing the run: the list carries the case total, and counting the finished ones there
+ * would be a per-case query for every run listed. The flow already records it — one slot per
+ * case in `flow_jobs_success`, null until that case's iteration is over. */
+ let caseProgress = $state>({})
+
+ async function readCaseProgress() {
+ const workspace = ws
+ const live = experiments.filter((e) => e.running)
+ if (!workspace || live.length === 0) {
+ if (Object.keys(caseProgress).length > 0) caseProgress = {}
+ return
+ }
+ const read = await Promise.all(
+ live.map(async (e) => {
+ try {
+ const update = await JobService.getJobUpdates({
+ workspace,
+ id: e.run_job_id,
+ running: true,
+ noLogs: true
+ })
+ const cases = update.flow_status?.modules?.find((m) => m.id === CASES_MODULE_ID)
+ if (!cases) return undefined
+ return [
+ e.id,
+ (cases.flow_jobs_success ?? []).filter((s) => s != undefined).length
+ ] as const
+ } catch {
+ // Left out of the map, so the row reads `0/total` until a later poll answers. A
+ // flow that cannot be read is already the list's problem to report, not this one's.
+ return undefined
+ }
+ })
+ )
+ caseProgress = Object.fromEntries(read.filter((e) => e !== undefined))
+ }
let experiment = $derived(experiments.find((e) => e.id === experimentId))
@@ -120,6 +172,7 @@
runsLoadError = false
try {
experiments = await listSubjectExperiments()
+ await readCaseProgress()
} catch (e) {
runsLoadError = true
sendUserToast(`Failed to load the runs: ${e}`, true)
@@ -183,6 +236,8 @@
// Switching datasets leaves the previous request in flight; only the newest may write, or a
// slow response for the dataset you just left replaces the one you are looking at.
let loadGeneration = 0
+ /** Which run the pane is opening; only the newest may clear `datasetLoading`. */
+ let openGeneration = 0
async function loadDataset(path: string | undefined): Promise {
const generation = ++loadGeneration
@@ -345,6 +400,7 @@
await loadResults()
} else {
experiments = await listSubjectExperiments()
+ await readCaseProgress()
}
} finally {
refreshing = false
@@ -354,27 +410,46 @@
/** Opens a run, bringing its dataset with it and offering the run before it as the baseline.
* Reading the cells is left to the effect on the selection, so every way in opens one alike. */
async function openRun(id: string) {
- // Against the dataset that is loaded, not the one that is selected: skipping on the selection
- // alone would leave a run open over a dataset whose cases and scorers were never read.
const target = experiments.find((e) => e.id === id)
- if (target && target.dataset !== dataset?.path) {
- await useDataset(target.dataset)
+ // Only when the run itself changes: re-showing the one already open — arrowing back into it
+ // from the list — must keep whatever comparison the user picked.
+ if (id !== experimentId) {
+ const index = experiments.findIndex((e) => e.id === id)
+ // The run before it *of the same dataset*: the list spans datasets, and a run of another
+ // set of cases is not a baseline for this one.
+ baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id
}
- const index = experiments.findIndex((e) => e.id === id)
- // The run before it *of the same dataset*: the list spans datasets, and a run of another set
- // of cases is not a baseline for this one.
- baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id
experimentId = id
- viewingRun = true
selectedCaseId = undefined
+ // Opened first, read second: the dataset is a request, and waiting on it here is a click
+ // that does nothing at all until the network answers. The page carries the wait instead.
+ //
+ // Against what is *selected* as well as what is loaded. `selectedDataset` moves the moment a
+ // read starts, so a load still in flight for another dataset shows up here: without that
+ // test, opening a run of the dataset already committed would skip `useDataset` entirely and
+ // leave the in-flight one free to commit its cases under this run.
+ const needsDataset =
+ !!target && (target.dataset !== dataset?.path || target.dataset !== selectedDataset)
+ datasetLoading = needsDataset
+ viewingRun = true
+ if (needsDataset) {
+ // Numbered like `loadDataset`'s own read, and for the same reason: opening a second run
+ // while the first is still loading leaves two `finally`s racing, and the loser clearing
+ // the flag would uncover the table with neither dataset in hand.
+ const generation = ++openGeneration
+ try {
+ await useDataset(target!.dataset)
+ } finally {
+ if (generation === openGeneration) datasetLoading = false
+ }
+ }
}
async function runAll(runSubject: EvalSubject, path: string): Promise {
if (!ws || !path) return false
running = true
- let id: string
try {
- id = await AiEvalsService.runExperiment({
+ await AiEvalsService.runExperiment({
workspace: ws,
requestBody: { dataset: path, subject: runSubject }
})
@@ -386,9 +461,10 @@
// From here the run exists and is billing: what can still fail is reading it back, and
// saying "failed to run" to that invites a second, duplicate run.
try {
+ // Onto the list rather than into the run: a run that has just started has no answers and
+ // no scores, and the list already fills its row in as they land. Reading it is a click.
if (path !== dataset?.path) await useDataset(path)
await loadRuns()
- await openRun(id)
} catch (e) {
sendUserToast(
`The run started but could not be read back: ${e}. Reload the runs list to see it.`,
@@ -407,8 +483,6 @@
/** The dataset is gone and every run of it with it: back to the list, on no dataset. */
async function datasetDeleted(path: string) {
- // A run dialog waiting behind the drawer has nothing to come back to.
- resumeRunDialog = false
if (selectedDataset === path) {
viewingRun = false
selectedCaseId = undefined
@@ -442,6 +516,13 @@
}
let selectedRow = $derived(displayRows.find((row) => row.case_id === selectedCaseId))
+ /** The case the side panel is showing. Held rather than read straight off the selection: the
+ * pane animates shut over a few hundred milliseconds, and the selection is gone on the first
+ * of them, which would empty the panel before it had finished closing. */
+ let openRow = $state(undefined)
+ $effect(() => {
+ if (selectedRow) openRow = selectedRow
+ })
$effect(() => {
if (!ws) return
@@ -570,34 +651,52 @@
+ {#if loaded && loadError}
+
+ Could not load evals
+
+ The datasets or runs could not be read. Check your access to this agent and reload.
+
+
+ {:else}
+
+ {
+ // Right opens the run under the highlight, falling back to whichever was open before;
+ // left is the way back, the same as the breadcrumb.
+ if (key === 'run') {
+ // Both branches go through `openRun`: it is what brings the run's own dataset back,
+ // and the fallback run may be of a dataset the list has since moved off.
+ const id = highlightedRunId ?? experimentId
+ if (id) openRun(id)
+ } else if (key === 'list') {
+ viewingRun = false
+ selectedCaseId = undefined
+ }
+ }}
+ pages={[
+ { key: 'list', content: listPage },
+ { key: 'run', content: runPage }
+ ]}
+ />
+ {/if}
+
+
+{#snippet listPage()}
+
+
+ Each run answers a dataset of cases with this agent and scores the answers, so runs can be
+ compared.
+
- Could not load evals
-
- The datasets or runs could not be read. Check your access to this agent and reload.
-
-
- {:else if loaded && datasets.length === 0}
-
- No dataset yet
-
- A dataset is the set of cases this agent is measured on. Runs are of a dataset, so
- it is the first thing to make.
-
- datasetDrawer?.openDrawer('new')}
- >
- New dataset
-
-
+{/snippet}
{
- if (await useDataset(path)) {
- resumeRunDialog = true
- datasetDrawer?.openDrawer('edit')
- }
- }}
- onNewDataset={() => {
- resumeRunDialog = true
- datasetDrawer?.openDrawer('new')
+ if (await useDataset(path)) datasetDrawer?.openDrawer('edit')
}}
+ onNewDataset={() => datasetDrawer?.openDrawer('new')}
/>
{
- if (!resumeRunDialog) return
- resumeRunDialog = false
- // On the dataset the drawer was just in: the dialog opens on the pane's own, which
- // creating or editing one has already moved to it.
- runDialogOpen = true
- }}
/>
+
+
diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css
deleted file mode 100644
index 1ac9f8b31c..0000000000
--- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css
+++ /dev/null
@@ -1,26 +0,0 @@
-/* MultilineCellEditor: a popup positioned over the cell, so it has to paint the cell's own frame
- rather than inherit it. */
-.ag-theme-alpine .wm-multiline-cell-editor,
-.ag-theme-alpine-dark .wm-multiline-cell-editor {
- background-color: var(--ag-background-color);
-}
-.ag-theme-alpine .wm-multiline-cell-editor textarea,
-.ag-theme-alpine-dark .wm-multiline-cell-editor textarea {
- display: block;
- box-sizing: border-box;
- /* Horizontal only: the vertical padding is set by the editor, which knows the height of the row
- it is replacing. `line-height` here is what it computes against. */
- padding: 0 calc(var(--ag-cell-horizontal-padding) - 1px);
- border: 1px solid var(--ag-input-focus-border-color);
- border-radius: 3px;
- outline: none;
- resize: none;
- /* Past this it scrolls rather than growing. */
- max-height: 40vh;
- overflow-y: auto;
- background-color: var(--ag-background-color);
- color: var(--ag-foreground-color);
- font: inherit;
- line-height: 20px;
- white-space: pre-wrap;
-}
diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts
deleted file mode 100644
index 00975955c9..0000000000
--- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts
+++ /dev/null
@@ -1,108 +0,0 @@
-import type { ColDef, ICellEditorComp, ICellEditorParams } from 'ag-grid-community'
-// Beside the editor rather than in the AgGrid theme: that file is the vendored theme, and a rule
-// added to it is one the next copy of it drops.
-import './multilineCellEditor.css'
-
-/** Kept in step with the `line-height` the stylesheet gives the textarea. */
-const LINE_HEIGHT = 20
-
-/**
- * A text cell editor that starts the height of the cell and grows as lines are added, for columns
- * holding prose rather than a value. Enter commits, Shift+Enter adds a line, Escape cancels.
- *
- * Rendered as a popup positioned over the cell: an in-cell editor is clipped to the row height, so
- * growing is only visible if the editor is allowed to paint outside it.
- */
-export class MultilineCellEditor implements ICellEditorComp {
- private eGui!: HTMLDivElement
- private textarea!: HTMLTextAreaElement
- private params!: ICellEditorParams
- private wasEmpty = false
-
- init(params: ICellEditorParams) {
- this.params = params
- this.eGui = document.createElement('div')
- this.eGui.className = 'wm-multiline-cell-editor'
-
- this.wasEmpty = params.value == undefined
-
- this.textarea = document.createElement('textarea')
- this.textarea.rows = 1
- // A keystroke that opened the edit replaces the value, as it does in every other cell; F2
- // and double-click keep it to be edited.
- this.textarea.value = params.eventKey?.length === 1 ? params.eventKey : (params.value ?? '')
- this.textarea.style.width = `${params.column.getActualWidth() - 2}px`
- // Padded so one line fills the cell it replaces and a second costs a line rather than a row.
- // From the row rather than from `--ag-row-height`, which is the theme's figure and not
- // necessarily this grid's.
- const rowHeight = params.node.rowHeight ?? 28
- const padding = Math.max(0, (rowHeight - LINE_HEIGHT - 2) / 2)
- this.textarea.style.paddingTop = `${padding}px`
- this.textarea.style.paddingBottom = `${padding}px`
-
- this.textarea.addEventListener('input', () => this.resize())
- this.textarea.addEventListener('keydown', (e) => {
- if (e.key === 'Escape') {
- // Kept from whatever is around the grid: a grid in a drawer or a dialog is under a
- // surface that closes on Escape, and leaving an edit is not asking to leave that.
- e.preventDefault()
- e.stopPropagation()
- this.params.api.stopEditing(true)
- return
- }
- if (e.key !== 'Enter' || e.isComposing) return
- // Both branches keep the key from the grid, which ends the edit on Enter whether or not
- // Shift is held: Shift+Enter falls through to the textarea's own newline, and plain Enter
- // ends the edit here instead.
- e.stopPropagation()
- if (!e.shiftKey) {
- e.preventDefault()
- this.params.stopEditing()
- }
- })
- this.eGui.appendChild(this.textarea)
- }
-
- private resize() {
- this.textarea.style.height = 'auto'
- this.textarea.style.height = `${this.textarea.scrollHeight}px`
- }
-
- getGui() {
- return this.eGui
- }
-
- afterGuiAttached() {
- this.resize()
- this.textarea.focus()
- // At the end rather than selected: a selection is a keystroke away from erasing the cell.
- const end = this.textarea.value.length
- this.textarea.setSelectionRange(end, end)
- }
-
- getValue() {
- // Nothing typed into a cell that held nothing is not an edit: returning '' here would write
- // an empty string over a null, which the grid would see as a change and commit.
- if (this.wasEmpty && this.textarea.value === '') return this.params.value
- return this.textarea.value
- }
-
- isPopup() {
- return true
- }
-
- getPopupPosition(): 'over' | 'under' {
- return 'over'
- }
-}
-
-/**
- * What a column of prose needs, ready to spread into a colDef. `suppressKeyboardEvent` as well as
- * the editor: the grid ends an edit on Enter from a handler a popup editor's DOM does not sit
- * under, so the editor cannot keep Shift+Enter for itself on its own.
- */
-export const multilineCellColDef: Pick = {
- cellEditor: MultilineCellEditor,
- suppressKeyboardEvent: (p) =>
- p.editing && (p.event as KeyboardEvent).key === 'Enter' && (p.event as KeyboardEvent).shiftKey
-}
diff --git a/frontend/src/lib/components/common/EditableTextarea.svelte b/frontend/src/lib/components/common/EditableTextarea.svelte
new file mode 100644
index 0000000000..26a6574027
--- /dev/null
+++ b/frontend/src/lib/components/common/EditableTextarea.svelte
@@ -0,0 +1,173 @@
+
+
+
+{#if editing}
+
+
+{:else}
+
+
+ {value || placeholder}
+
+{/if}
diff --git a/frontend/src/lib/components/common/drawer/Drawer.svelte b/frontend/src/lib/components/common/drawer/Drawer.svelte
index d375bf506e..6bb5358940 100644
--- a/frontend/src/lib/components/common/drawer/Drawer.svelte
+++ b/frontend/src/lib/components/common/drawer/Drawer.svelte
@@ -8,6 +8,7 @@
import { onMount, createEventDispatcher, setContext, untrack } from 'svelte'
import { BROWSER } from 'esm-env'
import Disposable from './Disposable.svelte'
+ import { setTopmostSurface } from '$lib/components/common/overlayHost.svelte'
import ConditionalPortal from './ConditionalPortal.svelte'
import { chatState } from '$lib/components/copilot/chat/sharedChatState.svelte'
import { useReducedMotion } from '$lib/svelte5Utils.svelte'
@@ -51,6 +52,11 @@
let disposable: Disposable | undefined = $state(undefined)
+ // A drawer stacks like a dialog does, so content inside it gets the same answer about whether
+ // its keys are meant for it. Without this, a drawer opened over a dialog would inherit the
+ // dialog's answer — false, because the drawer itself is now on top — and go deaf.
+ setTopmostSurface(() => disposable?.isTopmost() ?? true)
+
let reducedMotion = useReducedMotion()
let duration = $derived(reducedMotion.val ? 0 : _duration)
let durationMs = $derived(duration * 1000)
diff --git a/frontend/src/lib/components/common/emptyState/EmptyState.svelte b/frontend/src/lib/components/common/emptyState/EmptyState.svelte
index 8715065ad0..49c3cd2b77 100644
--- a/frontend/src/lib/components/common/emptyState/EmptyState.svelte
+++ b/frontend/src/lib/components/common/emptyState/EmptyState.svelte
@@ -10,6 +10,16 @@
label: string
icon?: any
onClick: () => void
+ /**
+ * `default` unless the surface has no other live call to action. Accent is for the case
+ * where this button is the only thing to press — a form whose submit is disabled until
+ * this is done, say — so it is not competing with one.
+ */
+ variant?: 'default' | 'accent'
+ /** Same write lock the surface's other controls take. An empty state is still a live
+ * control: without this it stays clickable while a request that has already read the
+ * empty list is in flight, and whatever it adds is discarded when that request lands. */
+ disabled?: boolean
aiId?: string
aiDescription?: string
}
@@ -32,11 +42,13 @@
{/if}
{#if action}
-
+
import { createBubbler, stopPropagation } from 'svelte/legacy'
import { ChevronLeft, ChevronRight } from 'lucide-svelte'
- import { getOverlayHost, overlayHostActive } from '$lib/components/common/overlayHost.svelte'
+ import {
+ getOverlayHost,
+ overlayHostActive,
+ setTopmostSurface
+ } from '$lib/components/common/overlayHost.svelte'
const bubble = createBubbler()
import { createEventDispatcher, untrack } from 'svelte'
@@ -38,6 +42,18 @@
/** A line under the title saying what the dialog is for; in the header so it does not
* scroll away with the body. */
description?: string
+ /** The body holds pages that are laid over each other rather than stacked, so the header
+ * keeps its own height and only the pages move. Requires `fillHeight`: pages are absolutely
+ * positioned and need a definite height to fill. Pair with `PagedContent`, which does the
+ * laying over; this only makes room for it. */
+ paginated?: boolean
+ /**
+ * Whether Enter confirms the dialog. On by default, which is right for a form. Turn it off
+ * where the body is a surface with its own meaning for Enter: the handler runs at `window`
+ * in the capture phase and stops propagation, so while it is on nothing inside the dialog
+ * can see the key at all.
+ */
+ enterConfirms?: boolean
/** Make the dialog fill the height it is anchored to and lay its body out as a flex
* column, so content can size itself with `h-full` / `flex-1 min-h-0`. Off by default:
* the dialog otherwise hugs its content, and percentage heights inside it do not
@@ -65,6 +81,8 @@
trail = undefined,
description = undefined,
fillHeight = false,
+ paginated = false,
+ enterConfirms = true,
minZIndex: minZIndexProp = undefined,
titleBadge,
settings,
@@ -95,6 +113,10 @@
// 1200 to stack above.
const minZIndex = $derived(minZIndexProp ?? (chatState.size > 0 ? zIndexes.aiChat + 1 : 0))
+ // So content in the body can tell a key meant for this dialog from one meant for whatever was
+ // opened over it. Read lazily: `disposable` is bound after this runs.
+ setTopmostSurface(() => disposable?.isTopmost() ?? true)
+
// Both `bind:open` and this $effect are needed: bind:open syncs the
// boolean, while the effect calls openDrawer/closeDrawer to register
// the disposable in the stacking system (same pattern as Drawer.svelte).
@@ -114,6 +136,7 @@
if (open) {
switch (event.key) {
case 'Enter':
+ if (!enterConfirms) break
event.stopPropagation()
event.preventDefault()
dispatch('confirmed')
@@ -231,7 +254,9 @@
{@render titleBadge?.()}
{/if}
{#if i > 0}
-
+
+
+
{/if}
{#if i === 0}
+
+ {#each pages as page, i (page.key)}
+ {#if visited.includes(page.key) || warmed}
+
+
+
+ {@render page.content()}
+
+ {/if}
+ {/each}
+
+
+
diff --git a/frontend/src/lib/components/common/overlayHost.svelte.ts b/frontend/src/lib/components/common/overlayHost.svelte.ts
index 0e76709cbf..7c4374115b 100644
--- a/frontend/src/lib/components/common/overlayHost.svelte.ts
+++ b/frontend/src/lib/components/common/overlayHost.svelte.ts
@@ -72,3 +72,23 @@ export function overlayHostActive(): () => boolean {
const host = getOverlayHost()
return () => host?.active() ?? true
}
+
+const TOPMOST_SURFACE_KEY = 'topmostSurface'
+
+/**
+ * Declare whether the surface enclosing this subtree is the one on top. Set by whatever owns the
+ * stacking — a dialog, a drawer — so content inside it can tell a key meant for itself from one
+ * meant for something opened over it.
+ */
+export function setTopmostSurface(isTopmost: () => boolean) {
+ setContext(TOPMOST_SURFACE_KEY, isTopmost)
+}
+
+/**
+ * Whether the enclosing surface is on top. True when nothing declared otherwise, so content that
+ * is not inside such a surface is not silently made deaf.
+ */
+export function topmostSurface(): () => boolean {
+ const isTopmost = getContext<(() => boolean) | undefined>(TOPMOST_SURFACE_KEY)
+ return () => isTopmost?.() ?? true
+}
diff --git a/frontend/src/lib/components/text_input/TextInput.svelte b/frontend/src/lib/components/text_input/TextInput.svelte
index c57f57120a..03fcedbd8e 100644
--- a/frontend/src/lib/components/text_input/TextInput.svelte
+++ b/frontend/src/lib/components/text_input/TextInput.svelte
@@ -81,6 +81,11 @@
size?: ButtonType.UnifiedSize
unifiedHeight?: boolean
underlyingInputEl?: UnderlyingInputElT
+ /**
+ * Passed to the `autosize` action on the `textarea` variant. Chiefly `minHeight: 0`, for a
+ * field that hugs one line instead of reserving the action's 30px floor.
+ */
+ autosizeParams?: import('$lib/autosize').AutosizeParams
}
export function focus() {
@@ -108,7 +113,8 @@
error,
size = 'md',
unifiedHeight = true,
- underlyingInputEl: _underlyingInputEl
+ underlyingInputEl: _underlyingInputEl,
+ autosizeParams
}: Props = $props()
let underlyingInputEl = $derived(_underlyingInputEl ?? ('input' as const))
@@ -152,7 +158,7 @@
onpointerdown={(e) => e.stopImmediatePropagation()}
bind:this={inputEl}
bind:value
- use:autosize
+ use:autosize={autosizeParams}
>
{:else if underlyingInputEl === 'input'}
Date: Mon, 31 Aug 2026 20:09:53 +0200
Subject: [PATCH 027/310] feat: free AI tokens + home search/filter revamp
(#10020)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat: add free Claude Opus tier with per-user token limit
Co-Authored-By: Claude Opus 4.8 (1M context)
* nit move alert
* Home AI Chat
* wire home ai chat
* auto send prompt
* refactor: remove keyboard arrow-navigation from home list
Co-Authored-By: Claude Opus 4.8 (1M context)
* feat: replace home search bar with unified FilterSearchbar
Co-Authored-By: Claude Opus 4.8 (1M context)
* feat: replace home quick tags with FilterSearchbar presets
Co-Authored-By: Claude Opus 4.8 (1M context)
* feat: add content filter to home FilterSearchbar with EE-gated content view
- Clear the kind filter by deleting the key (was showing a 'kind: null' tag on All)
- Remove the standalone Content button
- Add a 'content' filter; when set, render the Ctrl-K content-search view
(ContentSearchInner) which shows text-match snippets and its own EE warning
Co-Authored-By: Claude Opus 4.8 (1M context)
* feat: disable home AI chat and prompt to configure AI when no model
Co-Authored-By: Claude Opus 4.8 (1M context)
* track cost instead of tokens
* nit
* fix: load copilot config on home so AI chat isn't wrongly gated
Co-Authored-By: Claude Opus 4.8 (1M context)
* Home page update
* nits
* example prompts
* nit
* feat: switch free AI tier to DeepSeek with daily cost budgets
Co-Authored-By: Claude Opus 4.8 (1M context)
* nit
* Move bottom buttons to HomeAIChat
* [ee] feat: surface free AI tier state and make its metering abort-proof
Makes the free Windmill AI tier legible to the user and closes an abuse hole.
Backend:
- AIConfig gains a response-only free_tier marker (skip_deserializing so a
client can't store a forged one via edit_copilot_config). get_copilot_info
keeps returning it once the grant is spent, so the client knows AI is off
because the grant ran out, not because nothing was configured.
- Per-user grant becomes one-time (migration drops the day key from
ai_free_token_usage); the daily table stays as the instance kill-switch.
- Reserve-then-reconcile metering (see EE commit) so a mid-stream disconnect
can no longer dodge the usage report and get metered zero.
Frontend:
- copilotInfo carries freeTier; model settings show a "Free" pill and a
usage meter that warns past 80%.
- The home chat and the session chat show a dedicated "you've used your free
Windmill AI, add your own API key" state instead of the generic
"no provider configured" one.
- A failed send re-fetches copilot_info so the exhausted state (and its
banner) appears live, without a page reload.
Bumps ee-repo-ref.txt to the matching EE commit.
Co-Authored-By: Claude Opus 4.8 (1M context)
* feat: free AI usage meter reusing the context-usage gauge
Show free-tier spend with the same gauge as context usage instead of a
bespoke block:
- Extract the meter+tooltip into a shared UsageMeter; ContextUsageIndicator
uses it, and a new FreeTierUsageIndicator renders it from
copilotInfo.freeTier. Placed in the session-chat toolbar and next to the
home-chat model settings; the old meter block in the model-settings
dropdown is removed (the "Free" pill stays).
- Hide the context-usage bar while on the free tier so the free meter takes
that slot.
- Refresh copilotInfo after every free-tier turn (AIChatManager finally) so
the meter advances live and the turn that exhausts the grant flips to the
exhausted state, instead of both only updating on reload. Gated to active
free-tier users, so it costs nothing for configured-key users.
Co-Authored-By: Claude Opus 4.8 (1M context)
* docs: fix stale free-tier comments after DeepSeek/cost rework
Co-Authored-By: Claude Opus 4.8 (1M context)
* feat: always show context bar, replace free-tier meter with usage banner
Co-Authored-By: Claude Opus 4.8 (1M context)
* nit
* fix: atomic free-tier budget reservation (ee ref + sqlx)
Co-Authored-By: Claude Opus 4.8 (1M context)
* fix: keep CLI/MCP and Hub buttons unblurred on AI chat hover
Co-Authored-By: Claude Opus 4.8 (1M context)
* Add back arrow nav
* nit
* nit
* fix: three review P1s in the home AI chat & search
- AIChatManager: refreshFreeTierUsage now bails unless the global copilot
state still belongs to the completing manager's workspace, so a warm
session finishing after a workspace switch can't reload its (background)
workspace over the active one's models/client/copilotWorkspace.
- HomeAIChat: block submission until the copilot config is loaded AND
enabled (new `canSend`), so a prompt submitted during the unknown-config
window isn't handed to a session that never sends it and silently lost.
The disabled overlay still gates on config-loaded to avoid a flash.
- ItemsList: the content-search reload effect now depends on $workspaceStore
so content results follow the active workspace instead of showing the
previous one's.
Co-Authored-By: Claude Opus 4.8 (1M context)
* [ee] fix: harden the three home-AI-chat/search P1s after deeper review
Follow-up to the previous P1 commit; sharper review found the earlier guards
insufficient:
- refreshFreeTierUsage now compares against the most-recently-*requested*
workspace (new copilotWorkspaceRequested in aiStore, set synchronously in
loadCopilot), not the last-*resolved* one — otherwise a warm session
finishing while a newer workspace's load is still in flight could win the
monotonic token and restore its stale workspace over the one being loaded.
- The content-search view is keyed by workspace ({#key $workspaceStore}) so a
switch remounts ContentSearchInner; late in-flight responses from the
previous workspace can no longer land in the new one's component.
Backend (EE, via ee-repo-ref bump to 03ef0eb): the free-tier reservation now
also prices the worst-case input cap (at the cache-miss rate), and
enforce_free_tier_body rejects oversized prompts and pins n=1 — so an aborted
large-prompt request can no longer dodge the input bill that reconciliation
would otherwise charge.
Co-Authored-By: Claude Opus 4.8 (1M context)
* fix: exclude service accounts from the free AI tier
Free-tier eligibility was keyed solely on authed.email. Workspace admins can
create and impersonate arbitrary service accounts (synthetic *.sa.wm.dev
identities), each of which would receive its own one-time grant — letting one
tenant mint many grants and drain the instance-wide daily allowance. Skip the
free-tier fallback for *.sa.wm.dev identities.
Co-Authored-By: Claude Opus 4.8 (1M context)
* fix: activate free AI tier when clearing a workspace provider
edit_copilot_config returned AIConfig::default() when the saved workspace
config had no providers and no instance config existed; the frontend applies
that response immediately, disabling AI even though the free-tier key is
available. A later get_copilot_info (on reload) returns the synthetic free-tier
config, so clearing a provider behaved inconsistently until reload. Give this
response path the same free-tier fallback as get_copilot_info.
Co-Authored-By: Claude Opus 4.8 (1M context)
* fix: gate the home AI composer behind the global-AI dev flag
The "Build with AI" composer starts a session and navigates to /sessions, which
lives behind the same wm_dev_global_ai dev gate as the global AI chat. With the
gate off (the default), /sessions renders only its gate message, SessionWrapper
never mounts, and the queued prompt is silently dropped. Hide the home entry
point behind isGlobalAiEnabled() so it isn't exposed before the sessions gate
opens.
Co-Authored-By: Claude Opus 4.8 (1M context)
* [ee] chore: bump ee-repo-ref for deepseek-v4-flash price/model fix
Points at the EE commit that pins deepseek-v4-flash and its real prices
(pico-precision accounting).
Co-Authored-By: Claude Opus 4.8 (1M context)
* [ee] fix: provable byte bound for the free-tier input cap (ee-repo-ref)
Bumps ee-repo-ref to the EE commit that caps the raw request body byte length
directly (token_count <= byte_count is provable), replacing the unsafe
body.len()/2 token estimate that high-entropy prompts could beat.
Co-Authored-By: Claude Opus 4.8 (1M context)
* nit isGlobalAiEnabled
* empty commit
* fix(frontend): address Codex review on free-tier / home filters
- P1: home filters now sync from the URL reactively, so browser Back/Forward
updates the chips, kind toggle and results (and clears keys dropped from the
URL) instead of leaving them stale until the next filter edit.
- Free-tier banner buttons drop deprecated Button props (size/color/border
variant) for unifiedSize + a supported variant.
- Condense refreshFreeTierUsage comments to a single race-condition constraint
beside the guard.
Co-Authored-By: Claude Opus 4.8
* fix(frontend): hide empty kind badge on draft-only scripts
A draft-only script can carry an empty `kind`, which still isn't 'script' so the
row rendered a blue badge whose only content was capitalize('') — an empty pill
left of the "Draft only" badge. Guard the badge on a non-empty kind.
Co-Authored-By: Claude Opus 4.8
* feat(frontend): animate home tree-view group expand/collapse
Wrap each owner group's children in ResizeTransitionWrapper so height changes
animate. A slide transition only animates the initial mount, but a freshly-opened
owner fetches its rows and passes through a transient empty state before they land
— the ResizeObserver animates that second growth too. Nested TreeViews inherit the
wrapper's context and skip their own, so one observer per top-level owner animates
the whole subtree.
Co-Authored-By: Claude Opus 4.8
* feat(frontend): FilterSearchbar boolean auto-set and string-filter presets
- A default-false boolean filter has only one useful value, so selecting it sets
true immediately instead of opening a true/false picker. A default-true boolean
(e.g. "Include library scripts") still shows the picker, where false is the
meaningful choice — expressed via a new optional `default` on the schema.
- A plain string filter now surfaces any presets targeting it (`:`)
as suggestions once selected, integrated into menuItems so keyboard nav works —
previously selecting e.g. "Owner" showed nothing.
Co-Authored-By: Claude Opus 4.8
* feat(frontend): home page toolbar and content-filter revamp
- "New" create-menu button (scripts/flows/apps/…) replaces the old Content button;
the search bar moves to the right of the toggle group.
- Restore the content filter dropped in a merge: a `content` searchbar filter swaps
the list for the full-text ContentSearchInner view (EE), aligned flush with -mx-2.
- Move the owner/group and label chips off the page into FilterSearchbar presets;
ownerFilter/labelFilter now derive from the searchbar keys (data layer unchanged).
- Move the list controls (select / tree view / expand-all / sort) inline into the
top row between the toggle group and search bar; add margin above the list.
- Beta tag on the home AI chat; a bit more bottom margin under it; tighten the gap
between the admin/tutorial banners and the list.
Co-Authored-By: Claude Opus 4.8
* fix(ai): pass the request body to the free-tier reservation
Thread the prompt body into resolve_free_tier_credentials so the free tier can size its
upfront reservation from the actual request length instead of a fixed worst case (EE
c2e248b), fixing normal chats being rejected as "too large". Updates the OSS stub signature
and bumps ee-repo-ref.
Co-Authored-By: Claude Opus 4.8
* fix(frontend): gate home Create/Import menu on edit permissions
The relocated CreateActionsMenu rendered unconditionally, so operators and users in
workspaces protected from direct deployment saw create/import actions they can't use.
Restore the original gate (!operator && showEditButtons, the latter from NoDirectDeployAlert).
Co-Authored-By: Claude Opus 4.8
* fix(frontend): address Codex review on filter searchbar
- P1: the boolean shortcut now goes through the same tag-insertion path as the normal
branch, so it removes the typed search segment instead of leaving it as a stray
free-text (_default_) term.
- Mark the Runs `show_future_jobs` filter default: true so selecting it opens the picker
(false is the meaningful choice) rather than being a no-op.
- Home owner/label presets now emit the canonical `key:\ value` form so the applied-preset
check matches after a reparse and can't re-offer a duplicate; update the suggestion
extraction to strip the leading separator.
- Replace deprecated Button props (size/spacingSize/color) on the relocated list controls
with unifiedSize.
- Fix stale comments: UsageMeter no longer claims a free-tier consumer; the home filter
schema comment describes presets, not the removed ListFilters/label badges.
Co-Authored-By: Claude Opus 4.8
* fix(frontend): boolean filter shortcut sets value canonically
The round-1 shortcut baked `true` into the tag text, which merged into a following tag
(e.g. `archived:\ truekind:\ flow`). Instead remove the typed segment, set the value, and
reparse so the text is rebuilt canonically — no lingering free-text and no merge.
Co-Authored-By: Claude Opus 4.8
* docs(ai): restate free-tier caller identity contract in the OSS stub; bump ee-repo-ref
Co-Authored-By: Claude Opus 4.8
* fix(frontend): keep flanking tags separate when boolean shortcut drops a segment
Joining `before`/`after` directly fused the tags a removed mid-segment sat between
(e.g. `kind:\ flowsummary:\ bar`). Join with a space; reparse then canonicalizes. Also
trims the comment to the essential constraint.
Co-Authored-By: Claude Opus 4.8
* chore(ai): update sqlx cache for free-tier daily-day queries; bump ee-repo-ref
The reserve/reconcile daily-usage queries now bind the reservation day (EE change); refresh
their offline query cache and point ee-repo-ref at the EE commit.
Co-Authored-By: Claude Opus 4.8
* fix(ai): activate free tier when instance ai_config has no provider
An instance ai_config row won precedence just by existing, so an empty {} (valid via global
settings / declarative config) suppressed the free-tier fallback and left AI disabled — even
though build_copilot_settings_state already treats it as unconfigured. Apply the same
has_providers() check to the instance config in the proxy and edit_copilot_config paths.
Also refresh the sqlx cache for the reservation ceiling change and bump ee-repo-ref.
Co-Authored-By: Claude Opus 4.8
* fix(frontend): migrate legacy Home filter URLs to the searchbar keys
The old Home UI stored free-text in `search`, owner scope in `filter`, and could write
`kind=all`; the generic searchbar sync uses `_default_`, `owner`, and a kind enum without
`all`. Rewrite those params once before the sync reads the URL so shared/bookmarked links
restore, and drop `kind=all` which would otherwise wedge later filter edits.
Co-Authored-By: Claude Opus 4.8
* fix(ai): empty instance config in get_copilot_info; label user-disabled Home AI
- get_copilot_info returned any existing instance ai_config row before the free-tier
fallback, so an empty {} disabled AI in the copilot-info UI even though the proxy now
serves the free tier. Apply the same has_providers() gate here.
- The Home chat overlay said "No AI provider is configured" when the user had disabled AI
in account settings (providers still present). Distinguish that state ("Windmill AI is
disabled in your account settings") as the docked chat does, and drop the misleading
workspace-config button in that case.
Co-Authored-By: Claude Opus 4.8
* chore(ai): drop redundant proxy service-account check; trim TreeView comment
The service-account exclusion now lives in the free-tier helper, so the proxy calls it
directly. Also condense the tree-view resize-transition comment to the essential reason.
Bumps ee-repo-ref.
Co-Authored-By: Claude Opus 4.8
* docs(frontend): the Home content filter is not EE-gated
ContentSearchInner loads the workspace's scripts/flows/apps/resources and matches their
contents client-side, so it works on any instance. Drop the misleading "(EE)" from the
filter label and the "EE indexer / off-EE fallback" comments.
Co-Authored-By: Claude Opus 4.8
* chore(ee): bump ee-repo-ref for free-tier pricing + exhaustion fixes
Co-Authored-By: Claude Opus 4.8
Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS
* fix(frontend): show disabled Home AI overlay statically, not on hover
The disabled-state overlay (reason + configure/add-key action) was opacity-0 and
pointer-events-none until group-hover, so keyboard and touch users saw an inert composer
with no visible remedy. Render it and the composer blur statically when disabled instead.
Also bumps ee-repo-ref for the trimmed free-tier comments.
Co-Authored-By: Claude Opus 4.8
Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS
* fix(frontend): give account-disabled Home AI overlay a recovery action
The account-disabled branch showed a reason but hid every action, on the mistaken premise
that account settings has no linkable route. It opens from the #user-settings hash (the
same one the sidebar Account menu uses), so link there. Bumps ee-repo-ref for the
free-tier fixes.
Co-Authored-By: Claude Opus 4.8
Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS
* fix(frontend): gate Home AI composer for operators; a11y and filter-sync fixes
- Home composer now uses prefersSessionHandoff($userStore?.operator) instead of
isGlobalAiEnabled(): operators reached this route and could submit a prompt into a
/sessions page that refuses them, silently dropping it. Also drops the leftover empty
header spacer div above the chat.
- HomeAIChat: mark the blurred/disabled subtrees inert so keyboard users can't tab into
the unreadable textarea (pointer-events-none didn't stop Tab).
- ItemsList: keep the role-dependent searchbar keys (include_library, only_user_folders)
in the schema unconditionally and toggle `hidden` instead, so useUrlSyncedFilterInstance
(which snapshots the key set once) still URL-syncs a key that first appears after a
workspace switch.
- Bumps ee-repo-ref for the indexer non-parquet build fix.
Co-Authored-By: Claude Opus 4.8
Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS
* fix(frontend): keep CLI/MCP connect row for operators; trim filter comment
The previous commit gated all of HomeAIChat behind the operator/session check, which also
removed the AI-independent CLI/MCP "Connect workspace" drawer that operators (and the
sessions-beta opt-out) had on main. Render HomeAIChat for the same audience as before
(isGlobalAiEnabled) and gate only the composer (title, input, examples, overlay) on
operator status inside the component; the connect row always shows. Also trims the
role-dependent filter-schema comment to the <=4 line rule.
Co-Authored-By: Claude Opus 4.8
Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS
* fix(frontend): reconnect Home keyboard navigation to the unified searchbar
The searchbar migration replaced the the ItemsList keyboard
handler keys off, so Arrow/Enter no longer drove the results list. Thread an `id` down to the
searchbar's contenteditable (via TaggedTextInput/FilterSearchbar `inputId`) so the handler and
the workspace-switch focus restoration find it again; read the caret through the Selection API
instead of an 's selectionStart/End; and stand the list's arrows down while the
searchbar's suggestion dropdown is open (tracked via onDropdownVisibleChange). In free-text
mode the searchbar no longer opens its dropdown on a bare arrow key, so an empty box passes
Arrow/Enter to the list as before.
Co-Authored-By: Claude Opus 4.8
Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS
* fix(frontend): stop searchbar Enter inserting a newline; idle typewriter for operators
- TaggedTextInput is a single-line filter input, so Enter now preventDefaults the
contenteditable's newline insertion (surrounding suggestion-select / list-open handlers
still run on bubble). Previously Enter with no row highlighted dropped a literal \n into
the query.
- HomeAIChat's placeholder typewriter effect now runs only while the composer is shown, so
it no longer loops forever driving an unrendered input for operators.
Co-Authored-By: Claude Opus 4.8
Claude-Session: https://claude.ai/code/session_01BSqa1iRxn9GUE9fegT7bDS
* chore: update ee-repo-ref to f2a31156ac08ecb02d89dbc66d72be58e9c877ff
This commit updates the EE repository reference after PR #652 was merged in windmill-ee-private.
Previous ee-repo-ref: e59b96a2eea5d1110b40c842f17b337ab051bdd3
New ee-repo-ref: f2a31156ac08ecb02d89dbc66d72be58e9c877ff
Automated by sync-ee-ref workflow.
---------
Co-authored-by: Claude Opus 4.8 (1M context)
Co-authored-by: windmill-internal-app[bot]
---
...64fc5e65995ce81e622c174a89befc1a527e5.json | 22 +
...23f37b8d452ea8e199ccc207da238368f1996.json | 23 +
...e6be4cc3adc6aa859c3dbaa57fa50b63741cf.json | 15 +
...fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json | 23 +
...7e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json | 15 +
backend/ee-repo-ref.txt | 2 +-
...0260622072905_ai_free_token_usage.down.sql | 2 +
.../20260622072905_ai_free_token_usage.up.sql | 19 +
backend/summarized_schema.txt | 2 +
backend/windmill-api/openapi.yaml | 18 +
backend/windmill-api/src/ai.rs | 279 +++++----
backend/windmill-api/src/ai_free_tier_oss.rs | 65 +++
backend/windmill-api/src/lib.rs | 3 +
backend/windmill-api/src/workspaces.rs | 58 +-
frontend/src/lib/aiStore.ts | 13 +-
.../src/lib/components/FilterSearchbar.svelte | 145 ++++-
.../src/lib/components/TaggedTextInput.svelte | 11 +-
.../lib/components/common/table/AppRow.svelte | 2 +-
.../components/common/table/FlowRow.svelte | 2 +-
.../components/common/table/RawAppRow.svelte | 2 +-
.../lib/components/common/table/Row.svelte | 2 +
.../components/common/table/ScriptRow.svelte | 6 +-
.../lib/components/copilot/chat/AIChat.svelte | 28 +-
.../copilot/chat/AIChatDisplay.svelte | 63 +++
.../copilot/chat/AIChatManager.svelte.ts | 30 +-
.../copilot/chat/AIChatModelSettings.svelte | 14 +
.../copilot/chat/ContextUsageIndicator.svelte | 28 +-
.../components/copilot/chat/UsageMeter.svelte | 39 ++
.../src/lib/components/copilot/loadCopilot.ts | 9 +
.../src/lib/components/home/HomeAIChat.svelte | 255 +++++++++
.../src/lib/components/home/ItemsList.svelte | 529 ++++++++++--------
.../lib/components/home/NoItemFound.svelte | 1 -
.../src/lib/components/home/TreeView.svelte | 205 +++----
.../src/lib/components/runs/runsFilter.ts | 4 +-
.../components/select/GenericDropdown.svelte | 15 +-
.../sessions/sessionSwitch.svelte.ts | 6 +-
.../src/routes/(root)/(logged)/+page.svelte | 67 +--
37 files changed, 1457 insertions(+), 565 deletions(-)
create mode 100644 backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json
create mode 100644 backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json
create mode 100644 backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json
create mode 100644 backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json
create mode 100644 backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json
create mode 100644 backend/migrations/20260622072905_ai_free_token_usage.down.sql
create mode 100644 backend/migrations/20260622072905_ai_free_token_usage.up.sql
create mode 100644 backend/windmill-api/src/ai_free_tier_oss.rs
create mode 100644 frontend/src/lib/components/copilot/chat/UsageMeter.svelte
create mode 100644 frontend/src/lib/components/home/HomeAIChat.svelte
diff --git a/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json
new file mode 100644
index 0000000000..4ae74ff53c
--- /dev/null
+++ b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT cost_nanos FROM ai_free_token_usage WHERE email = $1",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "cost_nanos",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5"
+}
diff --git a/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json
new file mode 100644
index 0000000000..2253944269
--- /dev/null
+++ b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, $1::bigint, now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = ai_free_token_daily_usage.cost_nanos + $1::bigint,\n updated_at = now()\n RETURNING cost_nanos",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "cost_nanos",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Date"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996"
+}
diff --git a/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json
new file mode 100644
index 0000000000..a741ee1c0d
--- /dev/null
+++ b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, GREATEST(0, $1::bigint), now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_daily_usage.cost_nanos + $1::bigint),\n updated_at = now()",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Date"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf"
+}
diff --git a/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json
new file mode 100644
index 0000000000..7078c32f1a
--- /dev/null
+++ b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, $2::bigint, now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = ai_free_token_usage.cost_nanos + $2::bigint,\n updated_at = now()\n RETURNING cost_nanos",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "cost_nanos",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Varchar",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956"
+}
diff --git a/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json
new file mode 100644
index 0000000000..32f7e8cf1f
--- /dev/null
+++ b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, GREATEST(0, $2::bigint), now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_usage.cost_nanos + $2::bigint),\n updated_at = now()",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Varchar",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f"
+}
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index 51788b0e1d..2bb93f6bd9 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-df60763d1f243b0048dfc3fe700bc026b257bea8
+f2a31156ac08ecb02d89dbc66d72be58e9c877ff
diff --git a/backend/migrations/20260622072905_ai_free_token_usage.down.sql b/backend/migrations/20260622072905_ai_free_token_usage.down.sql
new file mode 100644
index 0000000000..95879034e4
--- /dev/null
+++ b/backend/migrations/20260622072905_ai_free_token_usage.down.sql
@@ -0,0 +1,2 @@
+DROP TABLE ai_free_token_daily_usage;
+DROP TABLE ai_free_token_usage;
diff --git a/backend/migrations/20260622072905_ai_free_token_usage.up.sql b/backend/migrations/20260622072905_ai_free_token_usage.up.sql
new file mode 100644
index 0000000000..ae0cf47e45
--- /dev/null
+++ b/backend/migrations/20260622072905_ai_free_token_usage.up.sql
@@ -0,0 +1,19 @@
+-- One-time grant of the Windmill-provided free AI tier, measured as cost in nano-dollars
+-- (1e-9 USD) rather than raw tokens — a prompt-cache hit costs a fraction of a fresh input
+-- token, so a token count wildly overstates the real bill. The grant never resets: once
+-- spent, the user must bring their own API key. Keyed by normalized email so the allowance
+-- is shared across a user's workspaces (and is resistant to +tag / gmail-dot aliasing).
+CREATE TABLE ai_free_token_usage (
+ email VARCHAR(255) PRIMARY KEY,
+ cost_nanos BIGINT NOT NULL DEFAULT 0,
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- Instance-wide daily cost ceiling (nano-dollars) for the free tier — a kill-switch
+-- independent of the per-user grant, bounding the blast radius of a bad day. One row per
+-- UTC day.
+CREATE TABLE ai_free_token_daily_usage (
+ day DATE PRIMARY KEY,
+ cost_nanos BIGINT NOT NULL DEFAULT 0,
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt
index 607ef99e0c..dfd0305e57 100644
--- a/backend/summarized_schema.txt
+++ b/backend/summarized_schema.txt
@@ -38,6 +38,8 @@ account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), clien
FK: (workspace_id) -> workspace(id)
agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char)
ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts)
+ai_free_token_daily_usage: day(date), cost_nanos(bigint), updated_at(ts)
+ai_free_token_usage: email(char), cost_nanos(bigint), updated_at(ts)
ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts)
FK: (workspace_id) -> workspace(id)
alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text)
diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml
index 09a3cf4195..31f0045ce0 100644
--- a/backend/windmill-api/openapi.yaml
+++ b/backend/windmill-api/openapi.yaml
@@ -27256,11 +27256,29 @@ components:
type: integer
minimum: 1
maximum: 2000000
+ free_tier:
+ $ref: "#/components/schemas/FreeTierInfo"
model_pricing:
type: object
additionalProperties:
$ref: "#/components/schemas/ModelPriceOverride"
+ FreeTierInfo:
+ type: object
+ description: >-
+ Read-only. Present when the workspace has no AI provider of its own and is running
+ on Windmill's free tier. Ignored on write.
+ properties:
+ exhausted:
+ type: boolean
+ description: The one-time grant is spent; no provider is served and the user must add their own API key.
+ used_ratio:
+ type: number
+ description: Fraction of the grant consumed, 0 to 1.
+ required:
+ - exhausted
+ - used_ratio
+
ModelPriceOverride:
type: object
description: negotiated rates in USD per million tokens, keyed `provider:model`
diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs
index 1a5f9dba87..c684830b27 100644
--- a/backend/windmill-api/src/ai.rs
+++ b/backend/windmill-api/src/ai.rs
@@ -409,6 +409,19 @@ impl ExpiringProviderCredentials {
}
}
+/// Set on the copilot config when the workspace has no AI provider of its own and is
+/// running on Windmill's free tier, so the client can label the lent model as free, warn
+/// before the grant runs out, and tell the user to add their own key once it has — rather
+/// than showing the same "no provider configured" state a never-configured workspace gets.
+#[derive(Serialize, Deserialize, Debug, Default, Clone)]
+pub struct FreeTierInfo {
+ /// The grant is spent: no provider is served and the user must bring their own key.
+ pub exhausted: bool,
+ /// Fraction of the grant consumed, 0.0..=1.0. A ratio, not a dollar amount — the
+ /// pricing model stays server-side.
+ pub used_ratio: f64,
+}
+
#[derive(Serialize, Deserialize, Debug, Default)]
pub struct AIConfig {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -423,6 +436,11 @@ pub struct AIConfig {
pub custom_prompts: Option>,
#[serde(skip_serializing_if = "Option::is_none")]
pub max_tokens_per_model: Option>,
+ /// Response-only: this same struct is the request body for saving a workspace's AI
+ /// config, and `skip_deserializing` is what stops a client from storing a forged
+ /// free-tier marker. Only the server sets it, per-request.
+ #[serde(skip_serializing_if = "Option::is_none", skip_deserializing)]
+ pub free_tier: Option,
/// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`.
/// Only models whose rates differ from the built-in table are stored.
#[serde(skip_serializing_if = "Option::is_none")]
@@ -1013,83 +1031,119 @@ async fn proxy(
check_scopes(&authed, || format!("resources:read:{}", resource_path))?;
}
- let mut credentials = match workspace_cache {
- Some(request_cache) if !request_cache.is_expired() && forced_resource_path.is_none() => {
- request_cache.credentials
- }
- _ => {
- let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) =
- if let Some(resource_path) = forced_resource_path {
- // forced resource path
- (resource_path, false, w_id.clone(), None)
- } else {
- let workspace_ai_config = sqlx::query_scalar!(
- "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1",
- &w_id
- )
- .fetch_one(&db)
- .await?;
+ // Set when serving the request through Windmill's free AI tier (the lent key). Holds
+ // the per-user concurrency lock and drives response metering.
+ let mut free_lease: Option = None;
+ let mut credentials = 'cred: {
+ match workspace_cache {
+ Some(request_cache)
+ if !request_cache.is_expired() && forced_resource_path.is_none() =>
+ {
+ request_cache.credentials
+ }
+ _ => {
+ let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) =
+ if let Some(resource_path) = forced_resource_path {
+ // forced resource path
+ (resource_path, false, w_id.clone(), None)
+ } else {
+ let workspace_ai_config = sqlx::query_scalar!(
+ "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1",
+ &w_id
+ )
+ .fetch_one(&db)
+ .await?;
- let (ai_config_value, resource_workspace, instance_ai_config_revision) = {
- let ws_has_config = workspace_ai_config
- .as_ref()
- .and_then(|v| serde_json::from_value::(v.clone()).ok())
- .is_some_and(|config| config.has_providers());
+ let (ai_config_value, resource_workspace, instance_ai_config_revision) = {
+ let ws_has_config = workspace_ai_config
+ .as_ref()
+ .and_then(|v| serde_json::from_value::(v.clone()).ok())
+ .is_some_and(|config| config.has_providers());
- if ws_has_config {
- (workspace_ai_config.unwrap(), w_id.clone(), None)
- } else {
- let instance_config = sqlx::query_scalar!(
- "SELECT value FROM global_settings WHERE name = 'ai_config'"
- )
- .fetch_optional(&db)
- .await?;
+ if ws_has_config {
+ (workspace_ai_config.unwrap(), w_id.clone(), None)
+ } else {
+ let instance_config = sqlx::query_scalar!(
+ "SELECT value FROM global_settings WHERE name = 'ai_config'"
+ )
+ .fetch_optional(&db)
+ .await?;
- match instance_config {
- Some(config) => (
- config,
- "admins".to_string(),
- Some(current_instance_ai_config_revision()),
- ),
- None => {
- return Err(Error::internal_err(
- "AI resource not configured".to_string(),
- ));
+ let instance_has_config =
+ instance_config.as_ref().is_some_and(|v| {
+ serde_json::from_value::(v.clone())
+ .ok()
+ .is_some_and(|c| c.has_providers())
+ });
+ match instance_config {
+ // An instance `ai_config` row with no usable provider (e.g. `{}`
+ // or `{"providers":{}}`) is treated as unconfigured, exactly as
+ // build_copilot_settings_state does — otherwise its mere presence
+ // would suppress the free-tier fallback below.
+ Some(config) if instance_has_config => (
+ config,
+ "admins".to_string(),
+ Some(current_instance_ai_config_revision()),
+ ),
+ _ => {
+ // Nothing configured: fall back to Windmill's free AI tier
+ // (EE-only) if a lent key is set and both the user's
+ // one-time grant and the instance's daily cap have room.
+ // Errors once the grant is spent, the day is capped, or the
+ // user already has a request in flight; None otherwise.
+ // Ineligible identities (e.g. service accounts) are refused
+ // inside the helper, so every path treats them alike.
+ let free =
+ crate::ai_free_tier_oss::resolve_free_tier_credentials(
+ &provider,
+ &db,
+ &ai_path,
+ &authed.email,
+ &body,
+ )
+ .await?;
+ if let Some((free_credentials, lease)) = free {
+ free_lease = Some(lease);
+ break 'cred free_credentials;
+ }
+ return Err(Error::internal_err(
+ "AI resource not configured".to_string(),
+ ));
+ }
}
}
+ };
+
+ let mut ai_config = serde_json::from_value::(ai_config_value)
+ .map_err(|e| Error::BadRequest(e.to_string()))?;
+
+ let provider_config = ai_config
+ .providers
+ .as_mut()
+ .and_then(|providers| providers.remove(&provider))
+ .ok_or_else(|| {
+ Error::BadRequest(format!("Provider {:?} not configured", provider))
+ })?;
+
+ if provider_config.resource_path.is_empty() {
+ return Err(Error::BadRequest("Resource path is empty".to_string()));
}
+
+ (
+ provider_config.resource_path,
+ true,
+ resource_workspace,
+ instance_ai_config_revision,
+ )
};
- let mut ai_config = serde_json::from_value::(ai_config_value)
- .map_err(|e| Error::BadRequest(e.to_string()))?;
-
- let provider_config = ai_config
- .providers
- .as_mut()
- .and_then(|providers| providers.remove(&provider))
- .ok_or_else(|| {
- Error::BadRequest(format!("Provider {:?} not configured", provider))
- })?;
-
- if provider_config.resource_path.is_empty() {
- return Err(Error::BadRequest("Resource path is empty".to_string()));
- }
-
- (
- provider_config.resource_path,
- true,
- resource_workspace,
- instance_ai_config_revision,
- )
- };
-
- // For user-specified resources, fetch through an RLS-scoped
- // connection so PostgreSQL row-level security enforces the same
- // folder/group boundaries as the regular resource API. For the
- // workspace/instance ai_config path, the resource_path was already
- // validated by an admin/devops user when configuring the workspace,
- // so the raw pool is used.
- let resource = if is_user_specified_resource {
+ // For user-specified resources, fetch through an RLS-scoped
+ // connection so PostgreSQL row-level security enforces the same
+ // folder/group boundaries as the regular resource API. For the
+ // workspace/instance ai_config path, the resource_path was already
+ // validated by an admin/devops user when configuring the workspace,
+ // so the raw pool is used.
+ let resource = if is_user_specified_resource {
let mut tx = user_db.clone().begin(&authed).await?;
let res = sqlx::query_scalar::<_, Option>>>(
"SELECT value FROM resource WHERE path = $1 AND workspace_id = $2",
@@ -1112,38 +1166,45 @@ async fn proxy(
.ok_or_else(|| Error::NotFound(format!("Could not find the resource {}, update the resource path in the workspace settings", resource_path)))?
.ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", resource_path)))?;
- let resource = serde_json::from_str::(resource.0.get())
- .map_err(|e| Error::BadRequest(e.to_string()))?;
+ let resource = serde_json::from_str::(resource.0.get())
+ .map_err(|e| Error::BadRequest(e.to_string()))?;
- // Enforce RLS on $var: resolution when the resource path was
- // user-specified (X-Resource-Path header) so users can only read
- // variables they have permission to access.
- let enforce_authed = if is_user_specified_resource {
- Some(&authed)
- } else {
- None
- };
- let credentials = resolve_provider_credentials(
- &provider,
- &db,
- &resource_workspace,
- resource,
- enforce_authed,
- )
- .await?;
- if save_to_cache {
- AI_REQUEST_CACHE.insert(
- (w_id.clone(), provider.clone()),
- ExpiringProviderCredentials::new(
- credentials.clone(),
- instance_ai_config_revision,
- ),
- );
+ // Enforce RLS on $var: resolution when the resource path was
+ // user-specified (X-Resource-Path header) so users can only read
+ // variables they have permission to access.
+ let enforce_authed = if is_user_specified_resource {
+ Some(&authed)
+ } else {
+ None
+ };
+ let credentials = resolve_provider_credentials(
+ &provider,
+ &db,
+ &resource_workspace,
+ resource,
+ enforce_authed,
+ )
+ .await?;
+ if save_to_cache {
+ AI_REQUEST_CACHE.insert(
+ (w_id.clone(), provider.clone()),
+ ExpiringProviderCredentials::new(
+ credentials.clone(),
+ instance_ai_config_revision,
+ ),
+ );
+ }
+ credentials
}
- credentials
}
};
+ // Free tier: pin the model and clamp max_tokens server-side before forwarding,
+ // since the request body is otherwise client-controlled.
+ if free_lease.is_some() {
+ body = crate::ai_free_tier_oss::enforce_free_tier_body(&body)?;
+ }
+
if let Some(fim_transform) =
maybe_transform_fim_request(&provider, &ai_path, &credentials.base_url, &body)?
{
@@ -1291,8 +1352,32 @@ async fn proxy(
let status_code = response.status();
let headers = response.headers().clone();
+ let is_sse = is_sse_response(&headers);
+
+ // Free tier: reconcile the cost reserved up-front against what the response actually
+ // used, holding the per-user lock (via the lease) until it is recorded. The chat
+ // streams (SSE), where the usage report only arrives in the final chunk; the
+ // non-streaming JSON path is handled for completeness.
+ if let Some(lease) = free_lease {
+ let body = if is_sse {
+ axum::body::Body::from_stream(inject_keepalives(
+ Box::pin(crate::ai_free_tier_oss::meter_usage(
+ response.bytes_stream(),
+ db.clone(),
+ lease,
+ )),
+ Duration::from_secs(KEEPALIVE_INTERVAL_SECS),
+ ))
+ } else {
+ let bytes = response.bytes().await.map_err(to_anyhow)?;
+ crate::ai_free_tier_oss::record_json_usage(db.clone(), lease, &bytes);
+ axum::body::Body::from(bytes)
+ };
+ return Ok((status_code, headers, body));
+ }
+
let stream = response.bytes_stream();
- let body = if is_sse_response(&headers) {
+ let body = if is_sse {
axum::body::Body::from_stream(inject_keepalives(
stream,
Duration::from_secs(KEEPALIVE_INTERVAL_SECS),
diff --git a/backend/windmill-api/src/ai_free_tier_oss.rs b/backend/windmill-api/src/ai_free_tier_oss.rs
new file mode 100644
index 0000000000..7a183ff8ed
--- /dev/null
+++ b/backend/windmill-api/src/ai_free_tier_oss.rs
@@ -0,0 +1,65 @@
+#[cfg(feature = "private")]
+#[allow(unused)]
+pub use crate::ai_free_tier_ee::*;
+
+// Open-source build: Windmill's free AI tier does not exist. These stubs make the
+// callers in `ai.rs` / `workspaces.rs` compile while disabling the feature entirely —
+// `resolve_free_tier_credentials` never opts in, so the proxy falls through to its
+// normal "AI resource not configured" path and the copilot stays hidden.
+//
+// Caller contract (enforced by the private impl, restated here for parity): the `email`
+// passed to `resolve_free_tier_credentials` / `free_tier_copilot_config` MUST be the
+// authenticated caller's own identity (an `ApiAuthed` email), never a client-supplied one —
+// it selects whose lent-key grant is spent and whose usage is read.
+
+#[cfg(not(feature = "private"))]
+use crate::ai::AIConfig;
+#[cfg(not(feature = "private"))]
+use crate::db::DB;
+#[cfg(not(feature = "private"))]
+use axum::body::Bytes;
+#[cfg(not(feature = "private"))]
+use windmill_ai::ai_providers::AIProvider;
+#[cfg(not(feature = "private"))]
+use windmill_ai::credentials::ProviderCredentials;
+#[cfg(not(feature = "private"))]
+use windmill_common::error::Result;
+
+#[cfg(not(feature = "private"))]
+pub struct FreeTierLease;
+
+#[cfg(not(feature = "private"))]
+pub async fn resolve_free_tier_credentials(
+ _provider: &AIProvider,
+ _db: &DB,
+ _ai_path: &str,
+ _email: &str,
+ _body: &Bytes,
+) -> Result