From d6476862b30692e450cceda09c58d47964f87d32 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Fri, 8 May 2026 04:05:13 +0000
Subject: [PATCH 01/17] fix(cli): forward HEADERS env var on every backend
 fetch call (#9075)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Several `fetch()` callers in the CLI bypassed `OpenAPI.HEADERS` and skipped
the `HEADERS` env var, causing requests to fail behind auth gateways like
Cloudflare Access (same shape as #6421):

- `pushScript()` `/scripts/create` and `/scripts/create_snapshot` — regressed
  in #8936 when the call switched from `wmill.createScript()` (SDK) to a raw
  `fetch` for the `skip_if_noop` query param.
- Script preview `/jobs/run/preview_bundle`.
- App dev `/jobs_u/getupdate_sse` SSE stream.
- `wmill docs` `/api/inkeep`.

All four now spread `getHeaders()` and call `detectAuthGatewayChallenge()`
so a Cloudflare/SSO challenge surfaces a clear error instead of an opaque
JSON parse failure.

Adds `test/headers_env_var.test.ts`: spins up an auth-gateway proxy that
403s requests missing `CF-Access-Client-Id` / `CF-Access-Client-Secret` and
otherwise reverse-proxies to the test backend, then runs `wmill sync push`
of a fresh script through the proxy. Negative case (no `HEADERS` env)
verifies the proxy actually gates; positive case asserts every request
including `/scripts/create` reaches the backend with the headers attached.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 cli/src/commands/app/dev.ts       |   7 +-
 cli/src/commands/docs/docs.ts     |   6 +
 cli/src/commands/script/script.ts |  20 ++-
 cli/test/headers_env_var.test.ts  | 258 ++++++++++++++++++++++++++++++
 4 files changed, 287 insertions(+), 4 deletions(-)
 create mode 100644 cli/test/headers_env_var.test.ts

diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts
index 5b78f84e3f..7edb793bea 100644
--- a/cli/src/commands/app/dev.ts
+++ b/cli/src/commands/app/dev.ts
@@ -14,7 +14,8 @@ import * as path from "node:path";
 import process from "node:process";
 import { Buffer } from "node:buffer";
 import { writeFileSync } from "node:fs";
-import { readTextFile } from "../../utils/utils.ts";
+import { getHeaders, readTextFile } from "../../utils/utils.ts";
+import { detectAuthGatewayChallenge } from "../../utils/http_guards.ts";
 import { WebSocket, WebSocketServer } from "ws";
 import {
   createFrameworkPlugins,
@@ -1714,13 +1715,17 @@ async function streamJobWithSSE(
   const sseUrl =
     `${baseUrl}api/w/${workspace}/jobs_u/getupdate_sse/${jobId}?fast=true`;
 
+  const extraHeaders = getHeaders();
   const response = await fetch(sseUrl, {
     headers: {
       Accept: "text/event-stream",
       Authorization: `Bearer ${token}`,
+      ...extraHeaders,
     },
   });
 
+  await detectAuthGatewayChallenge(response, sseUrl);
+
   if (!response.ok) {
     throw new Error(
       `SSE request failed: ${response.status} ${response.statusText}`,
diff --git a/cli/src/commands/docs/docs.ts b/cli/src/commands/docs/docs.ts
index d86d4ca367..fed91201e0 100644
--- a/cli/src/commands/docs/docs.ts
+++ b/cli/src/commands/docs/docs.ts
@@ -4,6 +4,8 @@ import * as log from "../../core/log.ts";
 import { requireLogin } from "../../core/auth.ts";
 import { resolveWorkspace } from "../../core/context.ts";
 import { GlobalOptions } from "../../types.ts";
+import { getHeaders } from "../../utils/utils.ts";
+import { detectAuthGatewayChallenge } from "../../utils/http_guards.ts";
 
 interface DocContentItem {
   title: string;
@@ -36,6 +38,7 @@ async function docs(
 
   console.log(colors.bold(`\nSearching Windmill docs...\n`));
 
+  const extraHeaders = getHeaders();
   let res: Response;
   try {
     res = await fetch(url, {
@@ -43,6 +46,7 @@ async function docs(
       headers: {
         "Content-Type": "application/json",
         Authorization: `Bearer ${workspace.token}`,
+        ...extraHeaders,
       },
       body: JSON.stringify({ query }),
     });
@@ -50,6 +54,8 @@ async function docs(
     throw new Error(`Network error connecting to ${workspace.remote}: ${e}`);
   }
 
+  await detectAuthGatewayChallenge(res, url);
+
   if (res.status === 403) {
     log.info(
       "Windmill documentation search is an Enterprise Edition feature. Please upgrade to use this command."
diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts
index 0635cd42ea..f1e605d8ff 100644
--- a/cli/src/commands/script/script.ts
+++ b/cli/src/commands/script/script.ts
@@ -12,7 +12,8 @@ import * as log from "../../core/log.ts";
 import { sep as SEP } from "node:path";
 import * as path from "node:path";
 import { stringify as yamlStringify } from "yaml";
-import { deepEqual, readTextFile, readTextFileSync } from "../../utils/utils.ts";
+import { deepEqual, getHeaders, readTextFile, readTextFileSync } from "../../utils/utils.ts";
+import { detectAuthGatewayChallenge } from "../../utils/http_guards.ts";
 import * as wmill from "../../../gen/services.gen.ts";
 import * as specificItems from "../../core/specific_items.ts";
 import { getCurrentGitBranch } from "../../utils/git.ts";
@@ -725,6 +726,7 @@ async function createScript(
   // (same content, lockfile, and metadata) as a no-op, so the CLI does not
   // produce phantom git-sync / promotion commits on re-pushes.
   const skipIfNoop = "skip_if_noop=true";
+  const extraHeaders = getHeaders();
   if (!bundleContent) {
     try {
       const url =
@@ -738,9 +740,11 @@ async function createScript(
         headers: {
           Authorization: `Bearer ${workspace.token}`,
           "Content-Type": "application/json",
+          ...extraHeaders,
         },
         body: JSON.stringify(body),
       });
+      await detectAuthGatewayChallenge(req, url);
       if (req.status != 201) {
         throw Error(
           `${req.status} - ${req.statusText} - ${await req.text()}`
@@ -771,9 +775,13 @@ async function createScript(
       skipIfNoop;
     const req = await fetch(url, {
       method: "POST",
-      headers: { Authorization: `Bearer ${workspace.token} ` },
+      headers: {
+        Authorization: `Bearer ${workspace.token} `,
+        ...extraHeaders,
+      },
       body: form,
     });
+    await detectAuthGatewayChallenge(req, url);
     if (req.status != 201) {
       throw Error(
         `Script snapshot creation was not successful: ${req.status} - ${
@@ -1587,12 +1595,18 @@ async function preview(
       workspace.workspaceId +
       "/jobs/run/preview_bundle";
 
+    const extraHeaders = getHeaders();
     const response = await fetch(url, {
       method: "POST",
-      headers: { Authorization: `Bearer ${workspace.token}` },
+      headers: {
+        Authorization: `Bearer ${workspace.token}`,
+        ...extraHeaders,
+      },
       body: form,
     });
 
+    await detectAuthGatewayChallenge(response, url);
+
     if (!response.ok) {
       throw new Error(
         `Preview failed: ${response.status} - ${response.statusText} - ${await response.text()}`
diff --git a/cli/test/headers_env_var.test.ts b/cli/test/headers_env_var.test.ts
new file mode 100644
index 0000000000..ad483f4d05
--- /dev/null
+++ b/cli/test/headers_env_var.test.ts
@@ -0,0 +1,258 @@
+/**
+ * Integration test: HEADERS env var is forwarded on every CLI fetch call.
+ *
+ * Spins up an auth-gateway proxy in front of the test backend that:
+ *   - 403s + Cloudflare-style HTML if the request is missing CF-Access-Client-Id /
+ *     CF-Access-Client-Secret (mirrors the real-world Cloudflare Access challenge),
+ *   - otherwise reverse-proxies to the backend.
+ *
+ * Then runs `wmill sync push` with --base-url pointed at the proxy. If any fetch
+ * in the CLI bypasses HEADERS, the proxy returns the challenge page and the push
+ * fails (or the request is logged as un-authenticated). Negative case verifies
+ * the gateway actually rejects un-headered requests, so a passing positive case
+ * is meaningful.
+ *
+ * Regression coverage for #6421 and the script.ts pushScript /
+ * jobs/run/preview_bundle / app dev SSE fetch calls that used to skip getHeaders().
+ */
+
+import { expect, test } from "bun:test";
+import { mkdir, writeFile } from "node:fs/promises";
+import type { Server } from "bun";
+import { withTestBackend } from "./test_backend.ts";
+
+const HEADER_NAMES = ["CF-Access-Client-Id", "CF-Access-Client-Secret"] as const;
+const HEADER_VALUES = {
+  "CF-Access-Client-Id": "test-cf-id",
+  "CF-Access-Client-Secret": "test-cf-secret",
+} as const;
+
+const HEADERS_ENV = HEADER_NAMES
+  .map((h) => `${h}: ${HEADER_VALUES[h]}`)
+  .join(", ");
+
+interface ProxyState {
+  authenticatedRequests: { method: string; path: string }[];
+  rejectedRequests: { method: string; path: string }[];
+}
+
+interface RunningProxy {
+  server: Server;
+  state: ProxyState;
+  url: string;
+}
+
+function startGatewayProxy(backendUrl: string): RunningProxy {
+  const state: ProxyState = {
+    authenticatedRequests: [],
+    rejectedRequests: [],
+  };
+
+  const server = Bun.serve({
+    port: 0,
+    hostname: "127.0.0.1",
+    async fetch(req) {
+      const reqUrl = new URL(req.url);
+      const pathAndQuery = reqUrl.pathname + reqUrl.search;
+
+      const id = req.headers.get("cf-access-client-id");
+      const secret = req.headers.get("cf-access-client-secret");
+      const passes =
+        id === HEADER_VALUES["CF-Access-Client-Id"] &&
+        secret === HEADER_VALUES["CF-Access-Client-Secret"];
+
+      if (!passes) {
+        state.rejectedRequests.push({ method: req.method, path: pathAndQuery });
+        return new Response(
+          '<!DOCTYPE html><title>Sign in ・ Cloudflare Access</title>' +
+            '<body>Authenticate to reach this site.</body>',
+          {
+            status: 403,
+            headers: {
+              "content-type": "text/html; charset=utf-8",
+              "cf-ray": "0000000000000000-TEST",
+              "cf-mitigated": "challenge",
+            },
+          },
+        );
+      }
+
+      state.authenticatedRequests.push({ method: req.method, path: pathAndQuery });
+
+      const target = new URL(pathAndQuery, backendUrl);
+      const forwardHeaders = new Headers(req.headers);
+      forwardHeaders.delete("cf-access-client-id");
+      forwardHeaders.delete("cf-access-client-secret");
+      forwardHeaders.set("host", new URL(backendUrl).host);
+
+      const body =
+        req.method === "GET" || req.method === "HEAD"
+          ? undefined
+          : await req.arrayBuffer();
+
+      return await fetch(target, {
+        method: req.method,
+        headers: forwardHeaders,
+        body,
+        redirect: "manual",
+      });
+    },
+  });
+
+  return {
+    server,
+    state,
+    url: `http://127.0.0.1:${server.port}`,
+  };
+}
+
+async function runCliThroughProxy(
+  backend: { workspace: string; testConfigDir: string; token?: string },
+  proxyUrl: string,
+  cliArgs: string[],
+  cwd: string,
+  env: Record<string, string>,
+): Promise<{ stdout: string; stderr: string; code: number }> {
+  const cliDir = new URL("..", import.meta.url).pathname;
+  const useNode = process.env["TEST_CLI_RUNTIME"] === "node";
+  const runtime = useNode ? "node" : "bun";
+  const entrypoint = useNode
+    ? `${cliDir}/npm/esm/main.js`
+    : `${cliDir}/src/main.ts`;
+  const runtimeArgs = useNode ? [entrypoint] : ["run", entrypoint];
+
+  const fullArgs = [
+    "--base-url", proxyUrl,
+    "--workspace", backend.workspace,
+    "--token", backend.token ?? "",
+    "--config-dir", backend.testConfigDir,
+    ...cliArgs,
+  ];
+
+  const proc = Bun.spawn([runtime, ...runtimeArgs, ...fullArgs], {
+    cwd,
+    env: { ...(process.env as Record<string, string>), ...env },
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+
+  const [stdout, stderr] = await Promise.all([
+    new Response(proc.stdout).text(),
+    new Response(proc.stderr).text(),
+  ]);
+  const code = await proc.exited;
+  return { stdout, stderr, code };
+}
+
+test(
+  "HEADERS env var is forwarded on every CLI fetch (sync push of new script)",
+  async () => {
+    await withTestBackend(async (backend, tempDir) => {
+      const proxy = startGatewayProxy(backend.baseUrl);
+      try {
+        await writeFile(
+          `${tempDir}/wmill.yaml`,
+          `defaultTs: bun\nincludes:\n  - "**"\nexcludes: []\n`,
+          "utf-8",
+        );
+
+        const uniqueId = Date.now();
+        const scriptName = `headers_${uniqueId}`;
+        const scriptDir = `${tempDir}/f/test`;
+        await mkdir(scriptDir, { recursive: true });
+        await writeFile(
+          `${scriptDir}/${scriptName}.ts`,
+          `export async function main() {\n  return "headers test ${uniqueId}";\n}\n`,
+          "utf-8",
+        );
+        await writeFile(
+          `${scriptDir}/${scriptName}.script.yaml`,
+          [
+            `summary: "headers regression"`,
+            `description: "Push covers /scripts/create + /jobs/run/dependencies_async"`,
+            `lock: ""`,
+            `schema:`,
+            `  $schema: "https://json-schema.org/draft/2020-12/schema"`,
+            `  type: object`,
+            `  properties: {}`,
+            `  required: []`,
+            `is_template: false`,
+            `kind: script`,
+            `language: bun`,
+            ``,
+          ].join("\n"),
+          "utf-8",
+        );
+
+        const includesGlob = `f/test/${scriptName}**`;
+
+        // Negative case: no HEADERS env -> the proxy should return the
+        // gateway challenge and the CLI should bail out non-zero. Proves the
+        // proxy is actually gating, so the positive case isn't a false pass.
+        const noHeaders = await runCliThroughProxy(
+          backend,
+          proxy.url,
+          ["sync", "push", "--yes", "--includes", includesGlob],
+          tempDir,
+          {},
+        );
+        expect(noHeaders.code).not.toEqual(0);
+        expect(proxy.state.rejectedRequests.length).toBeGreaterThan(0);
+        expect(proxy.state.authenticatedRequests.length).toEqual(0);
+
+        // Reset proxy state between cases.
+        proxy.state.authenticatedRequests.length = 0;
+        proxy.state.rejectedRequests.length = 0;
+
+        // Positive case: HEADERS env set -> the proxy must see those headers
+        // on every request the CLI makes for sync push to succeed.
+        const withHeaders = await runCliThroughProxy(
+          backend,
+          proxy.url,
+          ["sync", "push", "--yes", "--includes", includesGlob],
+          tempDir,
+          { HEADERS: HEADERS_ENV },
+        );
+        expect(withHeaders.code).toEqual(0);
+        expect(proxy.state.rejectedRequests).toEqual([]);
+
+        const paths = proxy.state.authenticatedRequests.map((r) => r.path);
+
+        // Print the full path list when an assertion below fails so the
+        // failure is debuggable without re-running with extra logging.
+        const debug = () => paths.join("\n  ");
+
+        // Tarball download (sync diff): src/commands/sync/pull.ts
+        expect(
+          paths.some((p) =>
+            p.startsWith(`/api/w/${backend.workspace}/workspaces/tarball`),
+          ),
+          `expected tarball request, got:\n  ${debug()}`,
+        ).toBe(true);
+
+        // Script create: src/commands/script/script.ts pushScript().
+        // This is the regression: PR #8936 switched from wmill.createScript()
+        // (SDK, inherits OpenAPI.HEADERS) to a raw fetch that didn't forward
+        // HEADERS. Without the fix, the proxy would 403 this request and the
+        // CLI exit would be non-zero — so this assertion is the load-bearing
+        // one for #8936 / #6421-style regressions.
+        expect(
+          paths.some((p) =>
+            p.includes(`/api/w/${backend.workspace}/scripts/create`),
+          ),
+          `expected /scripts/create request, got:\n  ${debug()}`,
+        ).toBe(true);
+
+        // Lock generation: src/utils/metadata.ts. Was the original #6421 fix
+        // (PR #6422) — a soft check; the backend may skip queueing a lock job
+        // for trivial bun scripts under some feature configurations, so we
+        // only verify it *if* the CLI tried to generate one.
+        // (No assertion — covered by the tarball+create checks above plus
+        // the negative case proving the proxy actually gates requests.)
+      } finally {
+        proxy.server.stop(true);
+      }
+    });
+  },
+  240_000,
+);

From bc527fd929577ac57d4e24196069ed236b702d71 Mon Sep 17 00:00:00 2001
From: hugocasa <hugo@casademont.ch>
Date: Fri, 8 May 2026 06:39:47 +0200
Subject: [PATCH 02/17] feat(cli): add --parallel flag to generate-metadata
 (#9074)

* feat(cli): add --parallel flag to generate-metadata

* fix(cli): validate --parallel input and harden flush ordering
---
 .../generate-metadata/generate-metadata.ts    | 255 +++++++++++-------
 cli/src/guidance/skills.gen.ts                |   2 +
 cli/src/utils/metadata.ts                     |  49 +++-
 .../auto-generated/cli/cli-commands.md        |   2 +
 system_prompts/auto-generated/prompts.ts      |   2 +
 .../skills/cli-commands/SKILL.md              |   2 +
 6 files changed, 209 insertions(+), 103 deletions(-)

diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts
index 226220592e..1de173e582 100644
--- a/cli/src/commands/generate-metadata/generate-metadata.ts
+++ b/cli/src/commands/generate-metadata/generate-metadata.ts
@@ -8,6 +8,8 @@ import { resolveWorkspace } from "../../core/context.ts";
 import { requireLogin } from "../../core/auth.ts";
 import * as log from "../../core/log.ts";
 import {
+  beginLockfileBatch,
+  flushLockfileBatch,
   generateScriptMetadataInternal,
   getRawWorkspaceDependencies,
   readLockfile,
@@ -200,6 +202,12 @@ export async function rehashOnly(
   const stubWorkspace = {} as any;
   const rehashOpts = { ...opts, rehashOnly: true } as any;
 
+  type RehashTask =
+    | { kind: "script"; scriptPath: string }
+    | { kind: "flow"; folder: string }
+    | { kind: "app"; folder: string; rawApp: boolean };
+  const queue: RehashTask[] = [];
+
   if (!rehashFilter?.skipScripts) {
     for (const e of scriptPaths) {
       // Filter against the derived remote path so a folder argument like
@@ -210,14 +218,7 @@ export async function rehashOnly(
       if (rehashFilter?.missingOnly) {
         if (skipIfExisting(remotePath) || skipIfExisting(remotePath, "__script_hash")) continue;
       }
-      try {
-        await generateScriptMetadataInternal(
-          e, stubWorkspace, rehashOpts, false, true, {}, codebases, false,
-        );
-        counts.scripts++;
-      } catch (err) {
-        log.warn(`Skipping ${e}: ${err instanceof Error ? err.message : err}`);
-      }
+      queue.push({ kind: "script", scriptPath: e });
     }
   }
 
@@ -228,12 +229,7 @@ export async function rehashOnly(
         const folderNormalized = f.replaceAll(SEP, "/");
         if (skipIfExisting(folderNormalized, "__flow_hash")) continue;
       }
-      try {
-        await generateFlowLockInternal(f, false, stubWorkspace, rehashOpts, false, true);
-        counts.flows++;
-      } catch (err) {
-        log.warn(`Skipping ${f}: ${err instanceof Error ? err.message : err}`);
-      }
+      queue.push({ kind: "flow", folder: f });
     }
   }
 
@@ -244,13 +240,52 @@ export async function rehashOnly(
         const folderNormalized = appFolder.replaceAll(SEP, "/");
         if (skipIfExisting(folderNormalized, "__app_hash")) continue;
       }
-      try {
-        await generateAppLocksInternal(appFolder, rawApp, false, stubWorkspace, rehashOpts, false, true);
-        counts.apps++;
-      } catch (err) {
-        log.warn(`Skipping ${appFolder}: ${err instanceof Error ? err.message : err}`);
+      queue.push({ kind: "app", folder: appFolder, rawApp });
+    }
+  }
+
+  let parallelism = Number(opts.parallel ?? 1);
+  if (!Number.isFinite(parallelism) || parallelism <= 0) parallelism = 1;
+  if (parallelism > 1) {
+    log.info(`Parallelizing ${parallelism} items at a time`);
+  }
+
+  // Buffer wmill-lock.yaml writes during the parallel phase: each task mutates
+  // the shared in-memory lockfile, then we flush once.
+  await beginLockfileBatch();
+  try {
+    const pool = new Set<Promise<void>>();
+    while (queue.length > 0 || pool.size > 0) {
+      while (pool.size < parallelism && queue.length > 0) {
+        const task = queue.shift()!;
+        const p = (async () => {
+          try {
+            if (task.kind === "script") {
+              await generateScriptMetadataInternal(
+                task.scriptPath, stubWorkspace, rehashOpts, false, true, {}, codebases, false,
+              );
+              counts.scripts++;
+            } else if (task.kind === "flow") {
+              await generateFlowLockInternal(task.folder, false, stubWorkspace, rehashOpts, false, true);
+              counts.flows++;
+            } else {
+              await generateAppLocksInternal(task.folder, task.rawApp, false, stubWorkspace, rehashOpts, false, true);
+              counts.apps++;
+            }
+          } catch (err) {
+            const label = task.kind === "script" ? task.scriptPath : task.folder;
+            log.warn(`Skipping ${label}: ${err instanceof Error ? err.message : err}`);
+          }
+        })();
+        pool.add(p);
+        p.then(() => pool.delete(p));
+      }
+      if (pool.size > 0) {
+        await Promise.race(pool);
       }
     }
+  } finally {
+    await flushLockfileBatch();
   }
 
   if (counts.scripts + counts.flows + counts.apps > 0 || !rehashFilter?.missingOnly) {
@@ -519,85 +554,117 @@ export async function generateMetadata(
 
   const errors: { path: string; error: string }[] = [];
 
-  // Process scripts
-  for (const item of scripts) {
-    current++;
-    log.info(`${formatProgress(current)} script ${item.path}`);
-    try {
-      await generateScriptMetadataInternal(
-        item.path, // originalPath with extension
-        workspace,
-        opts,
-        false, // dryRun
-        true, // noStaleMessage
-        mismatchedWorkspaceDeps,
-        codebases,
-        false,
-        tree
-      );
-    } catch (e) {
-      const msg = e instanceof Error ? e.message : String(e);
-      errors.push({ path: item.path, error: msg });
-      log.error(`  Failed: ${msg}`);
-    }
+  let parallelism = Number(opts.parallel ?? 1);
+  if (!Number.isFinite(parallelism) || parallelism <= 0) parallelism = 1;
+  if (parallelism > 1) {
+    log.info(`Parallelizing ${parallelism} items at a time`);
   }
 
-  // Process flows
-  for (const item of flows) {
-    current++;
-    try {
-      const result = await generateFlowLockInternal(
-        item.folder.replaceAll("/", SEP),
-        false, // dryRun
-        workspace,
-        opts,
-        false,
-        true, // noStaleMessage
-        tree
-      );
-      const flowResult = result as FlowLocksResult | undefined;
-      const scriptsInfo = flowResult?.updatedScripts?.length
-        ? colors.dim(colors.white(`: ${flowResult.updatedScripts.join(", ")}`))
-        : "";
-      log.info(`${formatProgress(current)} flow   ${item.path}${scriptsInfo}`);
-    } catch (e) {
-      const msg = e instanceof Error ? e.message : String(e);
-      errors.push({ path: item.path, error: msg });
-      log.info(`${formatProgress(current)} flow   ${item.path}`);
-      log.error(`  Failed: ${msg}`);
-    }
-  }
+  type Task =
+    | { kind: "script"; item: StaleItem }
+    | { kind: "flow"; item: StaleItem }
+    | { kind: "app"; item: StaleItem };
+  const queue: Task[] = [
+    ...scripts.map<Task>((item) => ({ kind: "script", item })),
+    ...flows.map<Task>((item) => ({ kind: "flow", item })),
+    ...apps.map<Task>((item) => ({ kind: "app", item })),
+  ];
 
-  // Process apps
-  for (const item of apps) {
-    current++;
-    try {
-      const result = await generateAppLocksInternal(
-        item.folder.replaceAll("/", SEP),
-        item.isRawApp!, // rawApp
-        false, // dryRun
-        workspace,
-        opts,
-        false,
-        true, // noStaleMessage
-        tree
-      );
-      const appResult = result as AppLocksResult | undefined;
-      const scriptsInfo = appResult?.updatedScripts?.length
-        ? colors.dim(colors.white(`: ${appResult.updatedScripts.join(", ")}`))
-        : "";
-      log.info(`${formatProgress(current)} app    ${item.path}${scriptsInfo}`);
-    } catch (e) {
-      const msg = e instanceof Error ? e.message : String(e);
-      errors.push({ path: item.path, error: msg });
-      log.info(`${formatProgress(current)} app    ${item.path}`);
-      log.error(`  Failed: ${msg}`);
+  // Buffer wmill-lock.yaml writes during the parallel phase: each task mutates
+  // the shared in-memory lockfile via clearGlobalLock/updateMetadataGlobalLock,
+  // then we flush once. Without this buffering, two workers' read-modify-write
+  // cycles would race and lose hashes.
+  await beginLockfileBatch();
+  try {
+    const pool = new Set<Promise<void>>();
+    while (queue.length > 0 || pool.size > 0) {
+      while (pool.size < parallelism && queue.length > 0) {
+        const task = queue.shift()!;
+        const taskNumber = ++current;
+        const p = (async () => {
+          if (task.kind === "script") {
+            const item = task.item;
+            log.info(`${formatProgress(taskNumber)} script ${item.path}`);
+            try {
+              await generateScriptMetadataInternal(
+                item.path, // originalPath with extension
+                workspace,
+                opts,
+                false, // dryRun
+                true, // noStaleMessage
+                mismatchedWorkspaceDeps,
+                codebases,
+                false,
+                tree
+              );
+            } catch (e) {
+              const msg = e instanceof Error ? e.message : String(e);
+              errors.push({ path: item.path, error: msg });
+              log.error(`  Failed: ${msg}`);
+            }
+          } else if (task.kind === "flow") {
+            const item = task.item;
+            try {
+              const result = await generateFlowLockInternal(
+                item.folder.replaceAll("/", SEP),
+                false, // dryRun
+                workspace,
+                opts,
+                false,
+                true, // noStaleMessage
+                tree
+              );
+              const flowResult = result as FlowLocksResult | undefined;
+              const scriptsInfo = flowResult?.updatedScripts?.length
+                ? colors.dim(colors.white(`: ${flowResult.updatedScripts.join(", ")}`))
+                : "";
+              log.info(`${formatProgress(taskNumber)} flow   ${item.path}${scriptsInfo}`);
+            } catch (e) {
+              const msg = e instanceof Error ? e.message : String(e);
+              errors.push({ path: item.path, error: msg });
+              log.info(`${formatProgress(taskNumber)} flow   ${item.path}`);
+              log.error(`  Failed: ${msg}`);
+            }
+          } else {
+            const item = task.item;
+            try {
+              const result = await generateAppLocksInternal(
+                item.folder.replaceAll("/", SEP),
+                item.isRawApp!, // rawApp
+                false, // dryRun
+                workspace,
+                opts,
+                false,
+                true, // noStaleMessage
+                tree
+              );
+              const appResult = result as AppLocksResult | undefined;
+              const scriptsInfo = appResult?.updatedScripts?.length
+                ? colors.dim(colors.white(`: ${appResult.updatedScripts.join(", ")}`))
+                : "";
+              log.info(`${formatProgress(taskNumber)} app    ${item.path}${scriptsInfo}`);
+            } catch (e) {
+              const msg = e instanceof Error ? e.message : String(e);
+              errors.push({ path: item.path, error: msg });
+              log.info(`${formatProgress(taskNumber)} app    ${item.path}`);
+              log.error(`  Failed: ${msg}`);
+            }
+          }
+        })();
+        pool.add(p);
+        p.then(() => pool.delete(p));
+      }
+      if (pool.size > 0) {
+        await Promise.race(pool);
+      }
     }
-  }
 
-  // Persist all stale workspace dep hashes (not just filtered — deps are global, not folder-scoped)
-  const allStaleDeps = staleItems.filter((i) => i.type === "dependencies");
-  await tree.persistDepsHashes(allStaleDeps.map((d) => d.path));
+    // Persist all stale workspace dep hashes (not just filtered — deps are global, not folder-scoped)
+    const allStaleDeps = staleItems.filter((i) => i.type === "dependencies");
+    await tree.persistDepsHashes(allStaleDeps.map((d) => d.path));
+  } finally {
+    await flushLockfileBatch();
+  }
 
   const succeeded = total - errors.length;
   log.info("");
@@ -640,6 +707,7 @@ const command = new Command()
   .option("--skip-flows", "Skip processing flows")
   .option("--skip-apps", "Skip processing apps")
   .option("--strict-folder-boundaries", "Only update items inside the specified folder (requires folder argument)")
+  .option("--parallel <n:number>", "Number of items to process in parallel")
   .option(
     "-i --includes <patterns:file[]>",
     "Comma separated patterns to specify which files to include"
@@ -661,6 +729,7 @@ const command = new Command()
       .option("--skip-scripts", "Skip processing scripts")
       .option("--skip-flows", "Skip processing flows")
       .option("--skip-apps", "Skip processing apps")
+      .option("--parallel <n:number>", "Number of items to process in parallel")
       .option(
         "-i --includes <patterns:file[]>",
         "Comma separated patterns to specify which files to include"
diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts
index 3eb1c872cc..ede19be85b 100644
--- a/cli/src/guidance/skills.gen.ts
+++ b/cli/src/guidance/skills.gen.ts
@@ -6730,6 +6730,7 @@ Generate metadata (locks, schemas) for all scripts, flows, and apps
 - \`--skip-flows\` - Skip processing flows
 - \`--skip-apps\` - Skip processing apps
 - \`--strict-folder-boundaries\` - Only update items inside the specified folder (requires folder argument)
+- \`--parallel <n:number>\` - Number of items to process in parallel
 - \`-i --includes <patterns:file[]>\` - Comma separated patterns to specify which files to include
 - \`-e --excludes <patterns:file[]>\` - Comma separated patterns to specify which files to exclude
 
@@ -6739,6 +6740,7 @@ Generate metadata (locks, schemas) for all scripts, flows, and apps
   - \`--skip-scripts\` - Skip processing scripts
   - \`--skip-flows\` - Skip processing flows
   - \`--skip-apps\` - Skip processing apps
+  - \`--parallel <n:number>\` - Number of items to process in parallel
   - \`-i --includes <patterns:file[]>\` - Comma separated patterns to specify which files to include
   - \`-e --excludes <patterns:file[]>\` - Comma separated patterns to specify which files to exclude
 
diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts
index 456336dae2..9668dd19e6 100644
--- a/cli/src/utils/metadata.ts
+++ b/cli/src/utils/metadata.ts
@@ -1202,7 +1202,32 @@ export function normalizeLockPath(p: string): string {
   return n;
 }
 
+// When set, `clearGlobalLock` and `updateMetadataGlobalLock` mutate this
+// in-memory copy instead of doing a full read-modify-write on disk for every
+// call. Callers that fan out item processing through a worker pool wrap the
+// pool with `beginLockfileBatch()`/`flushLockfileBatch()` so the lockfile is
+// only written once at the end — see `generate-metadata` parallelism.
+let inMemoryLock: Lock | null = null;
+
+export async function beginLockfileBatch(): Promise<void> {
+  if (inMemoryLock) return;
+  inMemoryLock = await readLockfile();
+}
+
+export async function flushLockfileBatch(): Promise<void> {
+  if (!inMemoryLock) return;
+  // Write first, then clear: if the disk write throws (e.g. ENOSPC), the
+  // buffered updates remain in memory and a retry can re-attempt the flush.
+  await writeFile(
+    WMILL_LOCKFILE,
+    yamlStringify(inMemoryLock as Record<string, any>, yamlOptions),
+    "utf-8",
+  );
+  inMemoryLock = null;
+}
+
 export async function readLockfile(): Promise<Lock> {
+  if (inMemoryLock) return inMemoryLock;
   let parsed: unknown;
   try {
     parsed = await yamlParseFile(WMILL_LOCKFILE);
@@ -1344,11 +1369,13 @@ export async function clearGlobalLock(path: string): Promise<void> {
         }
       });
     }
-    await writeFile(
-      WMILL_LOCKFILE,
-      yamlStringify(conf as Record<string, any>, yamlOptions),
-      "utf-8"
-    );
+    if (!inMemoryLock) {
+      await writeFile(
+        WMILL_LOCKFILE,
+        yamlStringify(conf as Record<string, any>, yamlOptions),
+        "utf-8"
+      );
+    }
   }
 }
 
@@ -1377,9 +1404,11 @@ export async function updateMetadataGlobalLock(
       conf.locks[path] = hash;
     }
   }
-  await writeFile(
-    WMILL_LOCKFILE,
-    yamlStringify(conf as Record<string, any>, yamlOptions),
-    "utf-8"
-  );
+  if (!inMemoryLock) {
+    await writeFile(
+      WMILL_LOCKFILE,
+      yamlStringify(conf as Record<string, any>, yamlOptions),
+      "utf-8"
+    );
+  }
 }
diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md
index e541df9198..692e2dd310 100644
--- a/system_prompts/auto-generated/cli/cli-commands.md
+++ b/system_prompts/auto-generated/cli/cli-commands.md
@@ -168,6 +168,7 @@ Generate metadata (locks, schemas) for all scripts, flows, and apps
 - `--skip-flows` - Skip processing flows
 - `--skip-apps` - Skip processing apps
 - `--strict-folder-boundaries` - Only update items inside the specified folder (requires folder argument)
+- `--parallel <n:number>` - Number of items to process in parallel
 - `-i --includes <patterns:file[]>` - Comma separated patterns to specify which files to include
 - `-e --excludes <patterns:file[]>` - Comma separated patterns to specify which files to exclude
 
@@ -177,6 +178,7 @@ Generate metadata (locks, schemas) for all scripts, flows, and apps
   - `--skip-scripts` - Skip processing scripts
   - `--skip-flows` - Skip processing flows
   - `--skip-apps` - Skip processing apps
+  - `--parallel <n:number>` - Number of items to process in parallel
   - `-i --includes <patterns:file[]>` - Comma separated patterns to specify which files to include
   - `-e --excludes <patterns:file[]>` - Comma separated patterns to specify which files to exclude
 
diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts
index aeb39bb6b7..83f6ae1157 100644
--- a/system_prompts/auto-generated/prompts.ts
+++ b/system_prompts/auto-generated/prompts.ts
@@ -2316,6 +2316,7 @@ Generate metadata (locks, schemas) for all scripts, flows, and apps
 - \`--skip-flows\` - Skip processing flows
 - \`--skip-apps\` - Skip processing apps
 - \`--strict-folder-boundaries\` - Only update items inside the specified folder (requires folder argument)
+- \`--parallel <n:number>\` - Number of items to process in parallel
 - \`-i --includes <patterns:file[]>\` - Comma separated patterns to specify which files to include
 - \`-e --excludes <patterns:file[]>\` - Comma separated patterns to specify which files to exclude
 
@@ -2325,6 +2326,7 @@ Generate metadata (locks, schemas) for all scripts, flows, and apps
   - \`--skip-scripts\` - Skip processing scripts
   - \`--skip-flows\` - Skip processing flows
   - \`--skip-apps\` - Skip processing apps
+  - \`--parallel <n:number>\` - Number of items to process in parallel
   - \`-i --includes <patterns:file[]>\` - Comma separated patterns to specify which files to include
   - \`-e --excludes <patterns:file[]>\` - Comma separated patterns to specify which files to exclude
 
diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md
index 66efee7a68..b4818b4fa1 100644
--- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md
+++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md
@@ -173,6 +173,7 @@ Generate metadata (locks, schemas) for all scripts, flows, and apps
 - `--skip-flows` - Skip processing flows
 - `--skip-apps` - Skip processing apps
 - `--strict-folder-boundaries` - Only update items inside the specified folder (requires folder argument)
+- `--parallel <n:number>` - Number of items to process in parallel
 - `-i --includes <patterns:file[]>` - Comma separated patterns to specify which files to include
 - `-e --excludes <patterns:file[]>` - Comma separated patterns to specify which files to exclude
 
@@ -182,6 +183,7 @@ Generate metadata (locks, schemas) for all scripts, flows, and apps
   - `--skip-scripts` - Skip processing scripts
   - `--skip-flows` - Skip processing flows
   - `--skip-apps` - Skip processing apps
+  - `--parallel <n:number>` - Number of items to process in parallel
   - `-i --includes <patterns:file[]>` - Comma separated patterns to specify which files to include
   - `-e --excludes <patterns:file[]>` - Comma separated patterns to specify which files to exclude
 

From 2067e0719fd1fd1b899b015badec0f222c054e66 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Fri, 8 May 2026 04:43:07 +0000
Subject: [PATCH 03/17] perf(flows): skip flow_env DB+transform work when no
 resolution is needed (#9078)

---
 backend/windmill-worker/src/common.rs      |  7 +++
 backend/windmill-worker/src/worker_flow.rs | 50 ++++++++++++++++------
 2 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs
index 0946075443..99e323dcfd 100644
--- a/backend/windmill-worker/src/common.rs
+++ b/backend/windmill-worker/src/common.rs
@@ -148,6 +148,13 @@ lazy_static::lazy_static! {
     static ref RE_RES_VAR: Regex = Regex::new(r#"\$(?:var|jsonvar|res|encrypted)\:"#).unwrap();
 }
 
+/// Returns true if any value in `vs` contains a `$var:`/`$jsonvar:`/`$res:`/`$encrypted:`
+/// reference that would need interpolation by `transform_json`. Cheap pre-check that
+/// callers can use to skip the DB roundtrip + clone path when nothing requires resolution.
+pub(crate) fn map_needs_resolution(vs: &HashMap<String, Box<RawValue>>) -> bool {
+    vs.values().any(|v| (*RE_RES_VAR).is_match(v.get()))
+}
+
 pub async fn transform_json<'a>(
     client: &AuthedClient,
     workspace: &str,
diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs
index 050ae4b44f..3a07d1bfe4 100644
--- a/backend/windmill-worker/src/worker_flow.rs
+++ b/backend/windmill-worker/src/worker_flow.rs
@@ -2394,21 +2394,47 @@ async fn resolve_flow_env_for_status_update(
     workspace_id: &str,
     flow_value: &FlowValue,
 ) -> Option<HashMap<String, Box<RawValue>>> {
-    let env = if let Some(ref e) = flow_value.flow_env {
-        e.clone()
-    } else {
-        fetch_root_flow_env(db, flow_job_id, workspace_id).await?
-    };
+    // Fetch the env source. For the inherited path, we first need to know whether the
+    // flow even has a parent — `fetch_root_flow_env` runs a recursive CTE on `v2_job`
+    // and is wasted work for top-level flows with no own `flow_env`. The mini job we
+    // pull here can be reused below if `transform_json` ends up needing it.
+    let (env, mini): (HashMap<String, Box<RawValue>>, Option<MiniPulledJob>) =
+        if let Some(ref e) = flow_value.flow_env {
+            (e.clone(), None)
+        } else {
+            let mini = match get_mini_pulled_job(db, &flow_job_id).await {
+                Ok(Some(j)) => j,
+                Ok(None) => return None,
+                Err(e) => {
+                    tracing::warn!("Failed to fetch flow job to resolve flow_env: {e:#}");
+                    return None;
+                }
+            };
+            if mini.parent_job.is_none() {
+                // No own flow_env and no parent to inherit from — nothing to resolve.
+                return None;
+            }
+            let env = fetch_root_flow_env(db, flow_job_id, workspace_id).await?;
+            (env, Some(mini))
+        };
     if env.is_empty() {
         return Some(env);
     }
-    let mini = match get_mini_pulled_job(db, &flow_job_id).await {
-        Ok(Some(j)) => j,
-        Ok(None) => return Some(env),
-        Err(e) => {
-            tracing::warn!("Failed to fetch flow job to resolve flow_env: {e:#}");
-            return Some(env);
-        }
+    // Skip the DB roundtrip + `transform_json` when nothing in env needs interpolation.
+    // This is the common case: a flow_env containing only literal values.
+    if !crate::common::map_needs_resolution(&env) {
+        return Some(env);
+    }
+    let mini = match mini {
+        Some(m) => m,
+        None => match get_mini_pulled_job(db, &flow_job_id).await {
+            Ok(Some(j)) => j,
+            Ok(None) => return Some(env),
+            Err(e) => {
+                tracing::warn!("Failed to fetch flow job to resolve flow_env: {e:#}");
+                return Some(env);
+            }
+        },
     };
     match transform_json(
         client,

From 1c56148714861aafc4f489916c71aa4674e938c0 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Fri, 8 May 2026 07:43:21 +0200
Subject: [PATCH 04/17] fix(cli-tests): stabilize flow lock-gen race + Windows
 path (#9080)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix(cli-tests): stabilize flow lock-gen race + Windows path

Three CLI test failures on the latest main, all flaky on CI:

1. `Mixed Case Paths: pull and push flow with capitalized folder` and
   `Integration: Mixed scripts and flows with nonDottedPaths are
   idempotent`: flow create/update queues an async FlowDependencies job
   that fills inline-script lockfiles and rewrites flow.value. The tests
   pulled/pushed before the worker finished, so dry-run idempotency saw
   phantom `*.inline_script.lock` adds and `flow.yaml` edits. Added a
   `waitForFlowDependencyJob` helper that polls `/flows/get` for the
   latest `dependency_job` and `/jobs_u/completed/get` until it lands,
   and called it after each API/CLI flow write in both tests.

2. `HEADERS env var is forwarded on every CLI fetch` (Windows-only,
   added in #9075): the new test built the CLI entrypoint via
   `new URL("..", import.meta.url).pathname`, which yields `/C:/...` on
   Windows and `Bun.spawn` rejected before reaching the proxy, leaving
   `rejectedRequests.length` at 0. Switched to
   `fileURLToPath` + `node:path.join` to match `cargo_backend.ts`.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fix(cli-tests): use /flows/deployment_status to actually wait for dep job

CI reviewers (Claude, Codex) flagged the prior `waitForFlowDependencyJob`
as a no-op: it read `flow.dependency_job` from `/api/w/{ws}/flows/get`,
but `Flow` / `FlowWithStarred` (backend/windmill-types/src/flows.rs:20-60)
do not include that field. The helper exited on the first iteration
without polling.

Switch to `/api/w/{ws}/flows/deployment_status/p/{path}`, which returns
`{ lock_error_logs, job_id }`. `job_id` is the FlowDependencies UUID
written into `deployment_metadata` in the same tx as the dep-job push
(backend/windmill-api-flows/src/flows.rs:660-672 and :1275-1292), so by
the time the create/update API call returns, the response carries the
latest dep-job UUID. Then poll `/jobs_u/completed/get/{job_id}` as
before. Local runtime for `mixed_case_paths.test.ts` jumps from ~9s to
~32s, confirming the helper now actually waits instead of returning
immediately. The 404 short-circuit in `sync_pull_push.test.ts` still
works — `get_deployment_status` returns 404 when the flow is absent.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 cli/test/headers_env_var.test.ts  | 12 +++++--
 cli/test/mixed_case_paths.test.ts | 55 +++++++++++++++++++++++++++++
 cli/test/sync_pull_push.test.ts   | 58 +++++++++++++++++++++++++++++++
 3 files changed, 122 insertions(+), 3 deletions(-)

diff --git a/cli/test/headers_env_var.test.ts b/cli/test/headers_env_var.test.ts
index ad483f4d05..72fdf4103f 100644
--- a/cli/test/headers_env_var.test.ts
+++ b/cli/test/headers_env_var.test.ts
@@ -18,6 +18,8 @@
 
 import { expect, test } from "bun:test";
 import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import type { Server } from "bun";
 import { withTestBackend } from "./test_backend.ts";
 
@@ -113,12 +115,16 @@ async function runCliThroughProxy(
   cwd: string,
   env: Record<string, string>,
 ): Promise<{ stdout: string; stderr: string; code: number }> {
-  const cliDir = new URL("..", import.meta.url).pathname;
+  // Use fileURLToPath + node:path so the CLI entrypoint is a real OS path on
+  // Windows. `new URL(..).pathname` yields `/C:/...` which Bun.spawn fails to
+  // resolve, so the negative case "rejected" on launch instead of reaching
+  // the proxy and the assertion on rejectedRequests > 0 flaked.
+  const cliDir = join(dirname(fileURLToPath(import.meta.url)), "..");
   const useNode = process.env["TEST_CLI_RUNTIME"] === "node";
   const runtime = useNode ? "node" : "bun";
   const entrypoint = useNode
-    ? `${cliDir}/npm/esm/main.js`
-    : `${cliDir}/src/main.ts`;
+    ? join(cliDir, "npm", "esm", "main.js")
+    : join(cliDir, "src", "main.ts");
   const runtimeArgs = useNode ? [entrypoint] : ["run", entrypoint];
 
   const fullArgs = [
diff --git a/cli/test/mixed_case_paths.test.ts b/cli/test/mixed_case_paths.test.ts
index d85f81cc83..872c6f94ff 100644
--- a/cli/test/mixed_case_paths.test.ts
+++ b/cli/test/mixed_case_paths.test.ts
@@ -130,6 +130,57 @@ async function createFlow(
     throw new Error(`Failed to create flow ${flowPath}: ${error}`);
   }
   await response.text();
+
+  // Flow creation queues an async FlowDependencies job that generates the
+  // inline-script lockfile and rewrites flow.value. If we don't wait, a
+  // subsequent pull/push races the worker and the dry-run idempotency
+  // assertion sees a phantom lock-add + flow.yaml-edit diff (CI-only flake).
+  await waitForFlowDependencyJob(backend, flowPath);
+}
+
+async function waitForFlowDependencyJob(
+  backend: any,
+  flowPath: string,
+  timeoutMs: number = 30000,
+): Promise<void> {
+  // /flows/get does not return `dependency_job`. The deployment_status route
+  // joins flow_version against deployment_metadata, which is populated in the
+  // same tx as the FlowDependencies push, so by the time the create/update
+  // API call returns, job_id is already the latest dep-job UUID.
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    const statusResp = await backend.apiRequest!(
+      `/api/w/${backend.workspace}/flows/deployment_status/p/${flowPath}`,
+    );
+    if (statusResp.status === 404) {
+      await statusResp.text().catch(() => {});
+      return;
+    }
+    if (!statusResp.ok) {
+      await statusResp.text().catch(() => {});
+      throw new Error(
+        `Failed to fetch deployment status for ${flowPath}: ${statusResp.status}`,
+      );
+    }
+    const status = await statusResp.json();
+    const depJobId: string | undefined = status?.job_id;
+    if (!depJobId) {
+      await new Promise((r) => setTimeout(r, 100));
+      continue;
+    }
+    const completed = await backend.apiRequest!(
+      `/api/w/${backend.workspace}/jobs_u/completed/get/${depJobId}`,
+    );
+    if (completed.ok) {
+      await completed.text();
+      return;
+    }
+    await completed.text().catch(() => {});
+    await new Promise((r) => setTimeout(r, 100));
+  }
+  throw new Error(
+    `Flow dependency job for ${flowPath} did not complete within ${timeoutMs}ms`,
+  );
 }
 
 async function createApp(
@@ -384,6 +435,10 @@ excludes: []
       const updatedFlow = await getFlow(backend, flowPath);
       expect(updatedFlow.summary).toEqual("Modified Data Processor Flow from test");
 
+      // The CLI push enqueues a fresh FlowDependencies job. Wait for it before
+      // the dry-run pull so the lock/flow.value writes have committed.
+      await waitForFlowDependencyJob(backend, flowPath);
+
       // Verify no diff on subsequent pull (idempotency)
       await verifyNoDiffOnPull(backend, tempDir);
     });
diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts
index e64af246f2..e31c4e1084 100644
--- a/cli/test/sync_pull_push.test.ts
+++ b/cli/test/sync_pull_push.test.ts
@@ -493,6 +493,58 @@ async function cleanupTempDir(dir: string): Promise<void> {
   }
 }
 
+// Polls /flows/deployment_status/p/{path} + /jobs_u/completed/get until the
+// most recent flow dependency job has completed. After a flow create/update
+// the API queues a FlowDependencies job that asynchronously fills inline-
+// script lockfiles and rewrites flow.value; tests that round-trip through
+// sync push/pull must wait for it or they race the worker (CI-only flake).
+// `deployment_status` is the right read here — `/flows/get` does not return
+// `dependency_job`, but the `deployment_metadata.job_id` row is written in
+// the same tx as the dep-job push, so the returned `job_id` is the latest
+// dep-job UUID by the time the create/update API call has returned. Returns
+// silently if the flow doesn't exist on the server (treats it as a no-op
+// push, since the route returns 404 when the flow row is missing).
+async function waitForFlowDependencyJob(
+  backend: any,
+  flowPath: string,
+  timeoutMs: number = 30000,
+): Promise<void> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    const statusResp = await backend.apiRequest!(
+      `/api/w/${backend.workspace}/flows/deployment_status/p/${flowPath}`,
+    );
+    if (statusResp.status === 404) {
+      await statusResp.text().catch(() => {});
+      return;
+    }
+    if (!statusResp.ok) {
+      await statusResp.text().catch(() => {});
+      throw new Error(
+        `Failed to fetch deployment status for ${flowPath}: ${statusResp.status}`,
+      );
+    }
+    const status = await statusResp.json();
+    const depJobId: string | undefined = status?.job_id;
+    if (!depJobId) {
+      await new Promise((r) => setTimeout(r, 100));
+      continue;
+    }
+    const completed = await backend.apiRequest!(
+      `/api/w/${backend.workspace}/jobs_u/completed/get/${depJobId}`,
+    );
+    if (completed.ok) {
+      await completed.text();
+      return;
+    }
+    await completed.text().catch(() => {});
+    await new Promise((r) => setTimeout(r, 100));
+  }
+  throw new Error(
+    `Flow dependency job for ${flowPath} did not complete within ${timeoutMs}ms`,
+  );
+}
+
 // =============================================================================
 // Tests
 // =============================================================================
@@ -1797,6 +1849,12 @@ excludes: []
 
       expect(pushResult.code).toEqual(0);
 
+      // sync push of the flow enqueues an async FlowDependencies job that
+      // generates the inline-script lockfile and rewrites flow.value. Wait for
+      // it before pulling back, otherwise pull races the worker and the
+      // dry-run push idempotency check sees phantom diffs (CI-only flake).
+      await waitForFlowDependencyJob(backend, flowName);
+
       // Pull back
       const pullResult = await backend.runCLICommand(
         ["sync", "pull", "--yes", "--includes", `f/test/mixed_*_${uniqueId}**`],

From e1a7c75e192b72b3b0d854c1901653e0b9386bf2 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Fri, 8 May 2026 07:43:46 +0200
Subject: [PATCH 05/17] perf(flows): cache resolved flow_env per flow execution
 (#9079)

* perf(flows): cache resolved flow_env per flow execution

* perf(flows): tighten flow_env cache cap to 1024 and clarify memory note

* perf(flows): don't cache transient flow_env resolution failures
---
 backend/Cargo.lock                         |  1 +
 backend/windmill-worker/Cargo.toml         |  1 +
 backend/windmill-worker/src/worker_flow.rs | 85 ++++++++++++++++++----
 3 files changed, 72 insertions(+), 15 deletions(-)

diff --git a/backend/Cargo.lock b/backend/Cargo.lock
index c6bf144004..323155c29b 100644
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -17895,6 +17895,7 @@ dependencies = [
  "process-wrap",
  "prometheus",
  "prost",
+ "quick_cache",
  "rand 0.9.0",
  "rcgen",
  "regex",
diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml
index ad3d87d8df..f1c7fa6c99 100644
--- a/backend/windmill-worker/Cargo.toml
+++ b/backend/windmill-worker/Cargo.toml
@@ -93,6 +93,7 @@ itertools.workspace = true
 regex.workspace = true
 prometheus = { workspace = true, optional = true }
 lazy_static.workspace = true
+quick_cache.workspace = true
 chrono.workspace = true
 dotenv.workspace = true
 rand.workspace = true # TODO: Remove. only used by token creation hack.
diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs
index 3a07d1bfe4..91c6cf9ec7 100644
--- a/backend/windmill-worker/src/worker_flow.rs
+++ b/backend/windmill-worker/src/worker_flow.rs
@@ -81,6 +81,33 @@ use windmill_audit::ActionKind;
 use windmill_common::audit::AuditAuthor;
 use windmill_queue::{canceled_job_to_result, push};
 
+lazy_static::lazy_static! {
+    /// Per-worker LRU cache of resolved `flow_env` values, keyed by flow job id.
+    /// `update_flow_status_after_job_completion_internal` runs once per child-step
+    /// completion, and re-resolving `$var:`/`$res:` references each time was the
+    /// dominant heap-allocation source under flow-heavy load. The cache aligns
+    /// predicate evaluation with `handle_flow`'s input-transform path, which
+    /// already resolves once at flow entry — predicates were the only place still
+    /// re-resolving on every read.
+    ///
+    /// Per-worker scope (caveat): the cache lives in process memory, not the DB,
+    /// so different workers processing children of the same flow each compute
+    /// their own snapshot on first miss. If a `$var:`/`$res:` value mutates
+    /// mid-flow, predicate eval on different workers can observe different
+    /// values for the same flow run. For typical use (env values configured at
+    /// flow start, read-only during execution) this is invisible. Cross-worker
+    /// determinism would require persisting the resolved env in `v2_job_status`;
+    /// see follow-up notes.
+    ///
+    /// Entries become dead weight once a flow completes and are evicted by LRU
+    /// pressure. Bounded at 1024 entries; per-entry footprint depends on the
+    /// env's contents (literals are small but a single resolved `$res:` can be
+    /// tens of KB), so worst-case memory scales with workload mix rather than
+    /// being fixed.
+    static ref RESOLVED_FLOW_ENV_CACHE: quick_cache::sync::Cache<Uuid, Arc<HashMap<String, Box<RawValue>>>> =
+        quick_cache::sync::Cache::new(1024);
+}
+
 #[derive(Debug)]
 pub struct SchedulePushZombieError(pub String);
 
@@ -480,8 +507,26 @@ pub async fn update_flow_status_after_job_completion_internal(
             .failure_module
             .as_ref()
             .is_some_and(|fm| retry_uses_flow_env(fm));
-        let resolved_flow_env: Option<HashMap<String, Box<RawValue>>> = if needs_flow_env {
-            resolve_flow_env_for_status_update(db, client, flow, w_id, flow_value).await
+        // The resolved env is constant for a flow's lifetime, so cache it by flow
+        // job id and reuse across child-step completions. Cache miss falls back to
+        // the existing resolve+persist path; same-worker subsequent completions are
+        // a single Arc::clone away. Transient-failure fallbacks (`is_cacheable ==
+        // false`) bypass the insert so a single API blip doesn't poison the rest
+        // of the flow run.
+        let resolved_flow_env: Option<Arc<HashMap<String, Box<RawValue>>>> = if needs_flow_env {
+            if let Some(cached) = RESOLVED_FLOW_ENV_CACHE.get(&flow) {
+                Some(cached)
+            } else {
+                resolve_flow_env_for_status_update(db, client, flow, w_id, flow_value)
+                    .await
+                    .map(|(env, is_cacheable)| {
+                        let arc = Arc::new(env);
+                        if is_cacheable {
+                            RESOLVED_FLOW_ENV_CACHE.insert(flow, arc.clone());
+                        }
+                        arc
+                    })
+            }
         } else {
             None
         };
@@ -632,7 +677,7 @@ pub async fn update_flow_status_after_job_completion_internal(
                         let bool_res = compute_bool_from_expr(
                             &expr,
                             Marc::new(args),
-                            resolved_flow_env.as_ref(),
+                            resolved_flow_env.as_deref(),
                             result.clone(),
                             all_iters,
                             id_ctx.as_ref(),
@@ -916,7 +961,7 @@ pub async fn update_flow_status_after_job_completion_internal(
                             &mut stop_early_err_msg,
                             &mut nresult,
                             args,
-                            resolved_flow_env.as_ref(),
+                            resolved_flow_env.as_deref(),
                             flow,
                             &old_status,
                         )
@@ -1136,7 +1181,7 @@ pub async fn update_flow_status_after_job_completion_internal(
                             &mut stop_early_err_msg,
                             &mut nresult,
                             args,
-                            resolved_flow_env.as_ref(),
+                            resolved_flow_env.as_deref(),
                             flow,
                             &old_status,
                         )
@@ -1219,7 +1264,7 @@ pub async fn update_flow_status_after_job_completion_internal(
                             &old_status.retry,
                             result.clone(),
                             Marc::new(args),
-                            resolved_flow_env.as_ref(),
+                            resolved_flow_env.as_deref(),
                             Some(client),
                         )
                         .await?
@@ -1601,7 +1646,7 @@ pub async fn update_flow_status_after_job_completion_internal(
                 &old_status.retry,
                 result.clone(),
                 Marc::new(args),
-                resolved_flow_env.as_ref(),
+                resolved_flow_env.as_deref(),
                 Some(client),
             )
             .await?
@@ -2387,13 +2432,19 @@ async fn fetch_root_flow_id(db: &DB, flow_id: Uuid) -> Uuid {
 // `update_flow_status_after_job_completion_internal`: take the current flow's
 // `flow_env` if present, otherwise inherit from the root flow, then interpolate
 // any `$var:`/`$res:` references via `transform_json`.
+//
+// Returns `Some((env, is_cacheable))`. `is_cacheable` is `false` only when the
+// returned env is a partially-resolved fallback after a transient error (e.g.
+// `transform_json` failed mid-resolve, or the mini job fetch failed). Callers
+// must not cache `is_cacheable == false` results — doing so would freeze the
+// transient failure for the rest of the flow run.
 async fn resolve_flow_env_for_status_update(
     db: &DB,
     client: &AuthedClient,
     flow_job_id: Uuid,
     workspace_id: &str,
     flow_value: &FlowValue,
-) -> Option<HashMap<String, Box<RawValue>>> {
+) -> Option<(HashMap<String, Box<RawValue>>, bool)> {
     // Fetch the env source. For the inherited path, we first need to know whether the
     // flow even has a parent — `fetch_root_flow_env` runs a recursive CTE on `v2_job`
     // and is wasted work for top-level flows with no own `flow_env`. The mini job we
@@ -2418,21 +2469,23 @@ async fn resolve_flow_env_for_status_update(
             (env, Some(mini))
         };
     if env.is_empty() {
-        return Some(env);
+        return Some((env, true));
     }
     // Skip the DB roundtrip + `transform_json` when nothing in env needs interpolation.
     // This is the common case: a flow_env containing only literal values.
     if !crate::common::map_needs_resolution(&env) {
-        return Some(env);
+        return Some((env, true));
     }
     let mini = match mini {
         Some(m) => m,
         None => match get_mini_pulled_job(db, &flow_job_id).await {
             Ok(Some(j)) => j,
-            Ok(None) => return Some(env),
+            // Don't cache: we couldn't fetch the job, so the env we'd return is the
+            // pre-interpolation literal — caching that would freeze the broken state.
+            Ok(None) => return Some((env, false)),
             Err(e) => {
                 tracing::warn!("Failed to fetch flow job to resolve flow_env: {e:#}");
-                return Some(env);
+                return Some((env, false));
             }
         },
     };
@@ -2445,11 +2498,13 @@ async fn resolve_flow_env_for_status_update(
     )
     .await
     {
-        Ok(Some(resolved)) => Some(resolved),
-        Ok(None) => Some(env),
+        Ok(Some(resolved)) => Some((resolved, true)),
+        Ok(None) => Some((env, true)),
+        // Don't cache transient resolution failures: one variable/resource API blip
+        // would otherwise poison the literal-only env for the rest of the flow run.
         Err(e) => {
             tracing::warn!("Failed to resolve flow_env references in status update: {e:#}");
-            Some(env)
+            Some((env, false))
         }
     }
 }

From 2510a097505c1bc290783fce2fc93b570e64ffa0 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Fri, 8 May 2026 08:16:21 +0200
Subject: [PATCH 06/17] chore(main): release 1.698.0 (#9076)

* chore(main): release 1.698.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
---
 CHANGELOG.md                                  |  19 +++
 backend/Cargo.lock                            | 156 +++++++++---------
 backend/Cargo.toml                            |   4 +-
 .../parsers/windmill-parser-wasm/Cargo.lock   |  48 +++---
 .../parsers/windmill-parser-wasm/Cargo.toml   |   2 +-
 backend/windmill-api/openapi.yaml             |   2 +-
 benchmarks/lib.ts                             |   2 +-
 cli/src/main.ts                               |   2 +-
 frontend/package-lock.json                    |   4 +-
 frontend/package.json                         |   2 +-
 lsp/Pipfile                                   |   2 +-
 openflow.openapi.yaml                         |   2 +-
 .../WindmillClient/WindmillClient.psd1        |   2 +-
 python-client/wmill/pyproject.toml            |   2 +-
 typescript-client/jsr.json                    |   2 +-
 typescript-client/package.json                |   2 +-
 version.txt                                   |   2 +-
 17 files changed, 137 insertions(+), 118 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a7f2f43e3a..1034bc4f87 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,24 @@
 # Changelog
 
+## [1.698.0](https://github.com/windmill-labs/windmill/compare/v1.697.0...v1.698.0) (2026-05-08)
+
+
+### Features
+
+* **cli:** add --parallel flag to generate-metadata ([#9074](https://github.com/windmill-labs/windmill/issues/9074)) ([bc527fd](https://github.com/windmill-labs/windmill/commit/bc527fd929577ac57d4e24196069ed236b702d71))
+
+
+### Bug Fixes
+
+* **cli-tests:** stabilize flow lock-gen race + Windows path ([#9080](https://github.com/windmill-labs/windmill/issues/9080)) ([1c56148](https://github.com/windmill-labs/windmill/commit/1c56148714861aafc4f489916c71aa4674e938c0))
+* **cli:** forward HEADERS env var on every backend fetch call ([#9075](https://github.com/windmill-labs/windmill/issues/9075)) ([d647686](https://github.com/windmill-labs/windmill/commit/d6476862b30692e450cceda09c58d47964f87d32))
+
+
+### Performance Improvements
+
+* **flows:** cache resolved flow_env per flow execution ([#9079](https://github.com/windmill-labs/windmill/issues/9079)) ([e1a7c75](https://github.com/windmill-labs/windmill/commit/e1a7c75e192b72b3b0d854c1901653e0b9386bf2))
+* **flows:** skip flow_env DB+transform work when no resolution is needed ([#9078](https://github.com/windmill-labs/windmill/issues/9078)) ([2067e07](https://github.com/windmill-labs/windmill/commit/2067e0719fd1fd1b899b015badec0f222c054e66))
+
 ## [1.697.0](https://github.com/windmill-labs/windmill/compare/v1.696.2...v1.697.0) (2026-05-07)
 
 
diff --git a/backend/Cargo.lock b/backend/Cargo.lock
index 323155c29b..47aff74297 100644
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -16025,7 +16025,7 @@ dependencies = [
 
 [[package]]
 name = "windmill"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -16106,7 +16106,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-ai"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "async-trait",
  "aws-config",
@@ -16130,7 +16130,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-alerting"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16143,7 +16143,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "argon2",
@@ -16286,7 +16286,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-agent-workers"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16309,7 +16309,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-assets"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16322,7 +16322,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-auth"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16348,7 +16348,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-client"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "reqwest 0.12.28",
  "serde",
@@ -16358,7 +16358,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-configs"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16375,7 +16375,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-debug"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "base64 0.22.1",
@@ -16397,7 +16397,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-embeddings"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16420,7 +16420,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-flow-conversations"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16436,7 +16436,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-flows"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16457,7 +16457,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-groups"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16478,7 +16478,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-inputs"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16492,7 +16492,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-integration-tests"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -16524,7 +16524,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-jobs"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16549,7 +16549,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-npm-proxy"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "flate2",
@@ -16567,7 +16567,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-openapi"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16589,7 +16589,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-schedule"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16609,7 +16609,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-scripts"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16639,7 +16639,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-settings"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16667,7 +16667,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-sse"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "lazy_static",
  "serde",
@@ -16679,7 +16679,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-users"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "argon2",
  "axum 0.8.4",
@@ -16704,7 +16704,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-workers"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16718,7 +16718,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-workspaces"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16751,7 +16751,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-audit"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "chrono",
  "lazy_static",
@@ -16765,7 +16765,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-autoscaling"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16784,7 +16784,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-common"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "aes-gcm",
  "aho-corasick",
@@ -16885,7 +16885,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-dep-map"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "chrono",
  "itertools 0.14.0",
@@ -16904,7 +16904,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-git-sync"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "regex",
  "serde",
@@ -16919,7 +16919,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-indexer"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "astral-tokio-tar",
@@ -16943,7 +16943,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-jseval"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "futures",
@@ -16960,7 +16960,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-macros"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "itertools 0.14.0",
  "lazy_static",
@@ -16976,7 +16976,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-mcp"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16997,7 +16997,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-native-triggers"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17028,7 +17028,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-oauth"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -17053,7 +17053,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-object-store"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -17087,7 +17087,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-operator"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "futures",
@@ -17105,7 +17105,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "convert_case 0.6.0",
  "serde",
@@ -17114,7 +17114,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -17126,7 +17126,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-csharp"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -17138,7 +17138,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "gosyn",
@@ -17150,7 +17150,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-graphql"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -17162,7 +17162,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-java"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -17174,7 +17174,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-nu"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "nu-parser",
@@ -17185,7 +17185,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-php"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -17196,7 +17196,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -17208,7 +17208,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-asset"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -17219,7 +17219,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -17241,7 +17241,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-r"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -17253,7 +17253,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ruby"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -17267,7 +17267,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-rust"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "convert_case 0.6.0",
@@ -17284,7 +17284,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -17297,7 +17297,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql-asset"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde",
@@ -17309,7 +17309,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -17327,7 +17327,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts-asset"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde-wasm-bindgen",
@@ -17343,7 +17343,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wac"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -17359,7 +17359,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-yaml"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde",
@@ -17370,7 +17370,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-queue"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -17407,7 +17407,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-runtime-nativets"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "const_format",
@@ -17445,7 +17445,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-sql-datatype-parser-wasm"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "getrandom 0.3.4",
  "wasm-bindgen",
@@ -17456,7 +17456,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-store"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -17486,7 +17486,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-test-utils"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17510,7 +17510,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17543,7 +17543,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-azure"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17576,7 +17576,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-email"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17596,7 +17596,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-gcp"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17630,7 +17630,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-http"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17666,7 +17666,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-kafka"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17689,7 +17689,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-mqtt"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17713,7 +17713,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-nats"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -17737,7 +17737,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-postgres"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17772,7 +17772,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-sqs"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17800,7 +17800,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-websocket"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17823,7 +17823,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-types"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "bitflags 2.9.4",
@@ -17842,7 +17842,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-once-cell",
@@ -17955,7 +17955,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker-volumes"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "bytes",
  "futures",
diff --git a/backend/Cargo.toml b/backend/Cargo.toml
index 0d25909c74..1a1b1f09d0 100644
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.697.0"
+version = "1.698.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -87,7 +87,7 @@ members = [
 exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
 
 [workspace.package]
-version = "1.697.0"
+version = "1.698.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock
index 4bb0074734..0bc102cc48 100644
--- a/backend/parsers/windmill-parser-wasm/Cargo.lock
+++ b/backend/parsers/windmill-parser-wasm/Cargo.lock
@@ -6183,7 +6183,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windmill-common"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "aho-corasick",
  "anyhow",
@@ -6263,7 +6263,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-macros"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6275,7 +6275,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "convert_case",
  "serde",
@@ -6284,7 +6284,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6296,7 +6296,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-csharp"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -6308,7 +6308,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "gosyn",
@@ -6320,7 +6320,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-graphql"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6332,7 +6332,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-java"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -6344,7 +6344,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-nu"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "nu-parser",
@@ -6355,7 +6355,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-php"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -6366,7 +6366,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -6378,7 +6378,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-asset"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -6389,7 +6389,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -6411,7 +6411,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-r"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -6423,7 +6423,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ruby"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6437,7 +6437,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-rust"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "convert_case",
@@ -6454,7 +6454,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6467,7 +6467,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql-asset"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde",
@@ -6479,7 +6479,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6497,7 +6497,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts-asset"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde-wasm-bindgen",
@@ -6513,7 +6513,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wac"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -6529,7 +6529,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wasm"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "getrandom 0.2.17",
@@ -6561,7 +6561,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-yaml"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "serde",
@@ -6572,7 +6572,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-types"
-version = "1.697.0"
+version = "1.698.0"
 dependencies = [
  "anyhow",
  "bitflags",
diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml
index e9dc39abc7..43a8bd59ef 100644
--- a/backend/parsers/windmill-parser-wasm/Cargo.toml
+++ b/backend/parsers/windmill-parser-wasm/Cargo.toml
@@ -12,7 +12,7 @@ resolver = "2"
 members = ["."]
 
 [workspace.package]
-version = "1.697.0"
+version = "1.698.0"
 edition = "2021"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 
diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml
index dd3c480241..0c8d63298e 100644
--- a/backend/windmill-api/openapi.yaml
+++ b/backend/windmill-api/openapi.yaml
@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.697.0
+  version: 1.698.0
   title: Windmill API
 
   contact:
diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts
index 2e1f9ef06a..a20c1b18af 100644
--- a/benchmarks/lib.ts
+++ b/benchmarks/lib.ts
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.697.0";
+export const VERSION = "v1.698.0";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({
diff --git a/cli/src/main.ts b/cli/src/main.ts
index 3d9a15dfd9..9650032679 100755
--- a/cli/src/main.ts
+++ b/cli/src/main.ts
@@ -79,7 +79,7 @@ export {
   token,
 };
 
-export const VERSION = "1.697.0";
+export const VERSION = "1.698.0";
 
 // Re-exported from constants.ts to maintain backwards compatibility
 export { WM_FORK_PREFIX } from "./core/constants.ts";
diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index 2e8ca86313..c616a8e212 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -1,12 +1,12 @@
 {
 	"name": "windmill-components",
-	"version": "1.697.0",
+	"version": "1.698.0",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "windmill-components",
-			"version": "1.697.0",
+			"version": "1.698.0",
 			"hasInstallScript": true,
 			"license": "AGPL-3.0",
 			"dependencies": {
diff --git a/frontend/package.json b/frontend/package.json
index 79d4465429..e6d623791f 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "windmill-components",
-	"version": "1.697.0",
+	"version": "1.698.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
diff --git a/lsp/Pipfile b/lsp/Pipfile
index 24a854d8e2..efe961555a 100644
--- a/lsp/Pipfile
+++ b/lsp/Pipfile
@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-wmill = ">=1.697.0"
+wmill = ">=1.698.0"
 sendgrid = "*"
 mysql-connector-python = "*"
 pymongo = "*"
diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml
index b4cd99fb13..d7e823897c 100644
--- a/openflow.openapi.yaml
+++ b/openflow.openapi.yaml
@@ -1,7 +1,7 @@
 openapi: '3.0.3'
 
 info:
-  version: 1.697.0
+  version: 1.698.0
   title: OpenFlow Spec
   contact:
     name: Ruben Fiszel
diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1
index 5626e3544d..2e25edb52d 100644
--- a/powershell-client/WindmillClient/WindmillClient.psd1
+++ b/powershell-client/WindmillClient/WindmillClient.psd1
@@ -12,7 +12,7 @@
     RootModule        = 'WindmillClient.psm1'
 
     # Version number of this module.
-    ModuleVersion     = '1.697.0'
+    ModuleVersion     = '1.698.0'
 
     # Supported PSEditions
     # CompatiblePSEditions = @()
diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml
index 868da1a79c..d1ac989649 100644
--- a/python-client/wmill/pyproject.toml
+++ b/python-client/wmill/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wmill"
-version = "1.697.0"
+version = "1.698.0"
 description = "A client library for accessing Windmill server wrapping the Windmill client API"
 license = "Apache-2.0"
 homepage = "https://windmill.dev"
diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json
index 135c0c7be3..07b5df0602 100644
--- a/typescript-client/jsr.json
+++ b/typescript-client/jsr.json
@@ -1,6 +1,6 @@
 {
   "name": "@windmill/windmill",
-  "version": "1.697.0",
+  "version": "1.698.0",
   "exports": "./src/index.ts",
   "publish": {
     "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"]
diff --git a/typescript-client/package.json b/typescript-client/package.json
index bbb6083085..566e4bd4e4 100644
--- a/typescript-client/package.json
+++ b/typescript-client/package.json
@@ -1,7 +1,7 @@
 {
   "name": "windmill-client",
   "description": "Windmill SDK client for browsers and Node.js",
-  "version": "1.697.0",
+  "version": "1.698.0",
   "author": "Ruben Fiszel",
   "license": "Apache 2.0",
   "sideEffects": false,
diff --git a/version.txt b/version.txt
index 20b63a5ebc..7185d214b0 100644
--- a/version.txt
+++ b/version.txt
@@ -1 +1 @@
-1.697.0
+1.698.0

From d37277d2341c83faf72efa0035cbf70e2cfbd596 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Fri, 8 May 2026 09:12:33 +0200
Subject: [PATCH 07/17] fix: reject root-rooted paths in ansible playbook
 validator on windows (#9081)

---
 .../windmill-worker/src/ansible_executor.rs   | 38 +++++++++++++------
 1 file changed, 27 insertions(+), 11 deletions(-)

diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs
index c8289b6934..2608cc6088 100644
--- a/backend/windmill-worker/src/ansible_executor.rs
+++ b/backend/windmill-worker/src/ansible_executor.rs
@@ -121,18 +121,24 @@ fn validate_relative_path(path: &str, field_name: &str) -> error::Result<()> {
         )));
     }
     let p = std::path::Path::new(trimmed);
-    if p.is_absolute() {
-        return Err(error::Error::BadRequest(format!(
-            "`{}` must be a relative path inside the cloned repo, got: {}",
-            field_name, trimmed
-        )));
-    }
     for component in p.components() {
-        if matches!(component, std::path::Component::ParentDir) {
-            return Err(error::Error::BadRequest(format!(
-                "`{}` must not contain `..` segments, got: {}",
-                field_name, trimmed
-            )));
+        match component {
+            // RootDir catches leading `/` or `\`; Prefix catches Windows drive
+            // letters and UNC paths. `Path::is_absolute()` alone misses
+            // RootDir-only paths on Windows (e.g. `/etc/passwd`).
+            std::path::Component::RootDir | std::path::Component::Prefix(_) => {
+                return Err(error::Error::BadRequest(format!(
+                    "`{}` must be a relative path inside the cloned repo, got: {}",
+                    field_name, trimmed
+                )));
+            }
+            std::path::Component::ParentDir => {
+                return Err(error::Error::BadRequest(format!(
+                    "`{}` must not contain `..` segments, got: {}",
+                    field_name, trimmed
+                )));
+            }
+            _ => {}
         }
     }
     Ok(())
@@ -1765,9 +1771,19 @@ mod tests {
 
     #[test]
     fn test_validate_relative_path_rejects_absolute() {
+        // `/etc/passwd` isn't `is_absolute()` on Windows (no drive prefix), but
+        // its leading RootDir still escapes the cloned repo, so reject it on
+        // every platform.
         assert!(validate_relative_path("/etc/passwd", "playbook").is_err());
     }
 
+    #[cfg(windows)]
+    #[test]
+    fn test_validate_relative_path_rejects_windows_absolute() {
+        assert!(validate_relative_path("\\etc\\passwd", "playbook").is_err());
+        assert!(validate_relative_path("C:\\Windows\\System32", "playbook").is_err());
+    }
+
     #[test]
     fn test_validate_relative_path_rejects_parent_dir() {
         assert!(validate_relative_path("../escape.yml", "playbook").is_err());

From ee3d82f01f52d835218f544dad6de9b7c3184fbb Mon Sep 17 00:00:00 2001
From: hugocasa <hugo@casademont.ch>
Date: Fri, 8 May 2026 09:59:46 +0200
Subject: [PATCH 08/17] fix(native-triggers): serialize Google channel renewal
 across replicas (#9060)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix(native-triggers): serialize Google channel renewal across replicas

`sync_all_triggers` runs every 5 minutes on every windmill-app replica
with no leader election. Multiple replicas were each rotating the
webhook token, creating a new Google watch channel, and racing the
trigger UPDATE — leaving the loser's new token (in `token`) and channel
(in Google) orphaned. Cloud was accumulating ~5 leaked tokens/week
without the silent best-effort `delete_token_by_hash` ever logging a
warning.

Wrap each per-trigger renewal in a transaction and acquire the row with
`SELECT … FOR UPDATE SKIP LOCKED`. Contending replicas skip the row
instead of duplicating the work. The lock spans `rotate_webhook_token`
→ Google API call → `update_native_trigger_service_config` and is only
released on commit. Re-checks `should_renew_channel` after acquiring
the lock so a replica that committed seconds earlier doesn't trigger a
duplicate renewal.

The pattern matches existing batch-cleanup paths in `monitor.rs`
(job-retention sweep) and other `FOR UPDATE SKIP LOCKED` call sites.

Also logs at `debug!` when `delete_token_by_hash` finds no matching row,
so future investigations can distinguish "deleted" from "not found"
without changing the `Ok(false)` contract.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fixup! fix(native-triggers): serialize Google channel renewal across replicas

* fixup! fix(native-triggers): serialize Google channel renewal across replicas

fixup! fix(native-triggers): serialize Google channel renewal across replicas

Address claude review:
- #5: per-skip log info -> debug (expected outcome under SKIP LOCKED)
- #2: warn moved out of delete_token_by_hash to the call site that knows the
  expected state (try_renew_channel_locked); other callers are race-prone and
  shouldn't warn
- #3: NULL service_config now warns (anomalous case)
- #4: post-Google-API DB-update + commit failures log distinctly so the
  channel-orphan case is grep-able

Plus: add 14d expiry to Google webhook tokens via ServiceName::webhook_token_expiration,
mint fresh ephemeral-webhook-{service}-{rd5} labels at create + rotate so the
existing 'ephemeral-' filter excludes them from user-token email/critical-alert
paths (no filter changes in 3 places). Orphans now self-clean via the existing
expiry sweep in monitor.rs.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fixup! fix(native-triggers): serialize Google channel renewal across replicas

fixup! fix(native-triggers): serialize Google channel renewal across replicas

Address second-round review:
- Claude #1 (P2): username_override_from_label now strips the 'ephemeral-'
  prefix for ephemeral-webhook-* labels, so created_by stays
  webhook-{service}-{rd5} instead of changing to label-ephemeral-webhook-...
  (preserves audit/job-list filter compatibility)
- Codex (P2): updated renew_channel doc — labels are no longer copied; rotate
  mints fresh ephemeral-webhook-google-{rd5} with 14d expiration
- Claude #3 (optional): test_rotate_webhook_token now asserts the rotated
  Google token has an ephemeral-webhook-google-* label and a populated
  expiration

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fixup! fix(native-triggers): serialize Google channel renewal across replicas

fixup! fix(native-triggers): serialize Google channel renewal across replicas

Reconsider the previous fixup: stripping the 'ephemeral-' prefix made
created_by no longer match token.label exactly, defeating the linking
purpose. Just allowlist 'ephemeral-webhook-' alongside the other
recognized webhook/email/ws prefixes — created_by becomes
ephemeral-webhook-google-XXXXX, matching token.label exactly. The
'ephemeral-' substring also informs operators that this is a
system-managed auto-expiring token vs a user-managed webhook trigger.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 ...3d94c27fcbbf989a4a70678dbc54e0d896e24.json |  41 ++++
 ...fb888d53c5e4aa53bfc43ca296863bc61813f.json |  46 +++++
 ...a2be62ce1f8f7edf3793d7ad4af9eb11b648f.json |  28 +++
 backend/windmill-api-auth/src/auth.rs         |   3 +-
 .../tests/token_hash.rs                       |  33 ++-
 .../src/google/external.rs                    | 190 +++++++++++++-----
 .../windmill-native-triggers/src/handler.rs   |  15 +-
 backend/windmill-native-triggers/src/lib.rs   |  57 ++++--
 8 files changed, 333 insertions(+), 80 deletions(-)
 create mode 100644 backend/.sqlx/query-6474dbc070afdce3e6bc74b02f93d94c27fcbbf989a4a70678dbc54e0d896e24.json
 create mode 100644 backend/.sqlx/query-676405f5ba49c1c711646bd0882fb888d53c5e4aa53bfc43ca296863bc61813f.json
 create mode 100644 backend/.sqlx/query-b02d3563b72ee8989dea128a3c6a2be62ce1f8f7edf3793d7ad4af9eb11b648f.json

diff --git a/backend/.sqlx/query-6474dbc070afdce3e6bc74b02f93d94c27fcbbf989a4a70678dbc54e0d896e24.json b/backend/.sqlx/query-6474dbc070afdce3e6bc74b02f93d94c27fcbbf989a4a70678dbc54e0d896e24.json
new file mode 100644
index 0000000000..702e6e36bb
--- /dev/null
+++ b/backend/.sqlx/query-6474dbc070afdce3e6bc74b02f93d94c27fcbbf989a4a70678dbc54e0d896e24.json
@@ -0,0 +1,41 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        SELECT service_config, webhook_token_hash\n        FROM native_trigger\n        WHERE workspace_id = $1\n          AND service_name = $2\n          AND external_id = $3\n        FOR UPDATE SKIP LOCKED\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "service_config",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 1,
+        "name": "webhook_token_hash",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        {
+          "Custom": {
+            "name": "native_trigger_service",
+            "kind": {
+              "Enum": [
+                "nextcloud",
+                "google",
+                "github"
+              ]
+            }
+          }
+        },
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      false
+    ]
+  },
+  "hash": "6474dbc070afdce3e6bc74b02f93d94c27fcbbf989a4a70678dbc54e0d896e24"
+}
diff --git a/backend/.sqlx/query-676405f5ba49c1c711646bd0882fb888d53c5e4aa53bfc43ca296863bc61813f.json b/backend/.sqlx/query-676405f5ba49c1c711646bd0882fb888d53c5e4aa53bfc43ca296863bc61813f.json
new file mode 100644
index 0000000000..a1aa4cc015
--- /dev/null
+++ b/backend/.sqlx/query-676405f5ba49c1c711646bd0882fb888d53c5e4aa53bfc43ca296863bc61813f.json
@@ -0,0 +1,46 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT email, scopes, workspace_id, super_admin, owner FROM token WHERE token_hash = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "scopes",
+        "type_info": "TextArray"
+      },
+      {
+        "ordinal": 2,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "super_admin",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 4,
+        "name": "owner",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "676405f5ba49c1c711646bd0882fb888d53c5e4aa53bfc43ca296863bc61813f"
+}
diff --git a/backend/.sqlx/query-b02d3563b72ee8989dea128a3c6a2be62ce1f8f7edf3793d7ad4af9eb11b648f.json b/backend/.sqlx/query-b02d3563b72ee8989dea128a3c6a2be62ce1f8f7edf3793d7ad4af9eb11b648f.json
new file mode 100644
index 0000000000..3536567e16
--- /dev/null
+++ b/backend/.sqlx/query-b02d3563b72ee8989dea128a3c6a2be62ce1f8f7edf3793d7ad4af9eb11b648f.json
@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT label, expiration FROM token WHERE token_hash = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "label",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "expiration",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true
+    ]
+  },
+  "hash": "b02d3563b72ee8989dea128a3c6a2be62ce1f8f7edf3793d7ad4af9eb11b648f"
+}
diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs
index d4c448e934..ac4017b81c 100644
--- a/backend/windmill-api-auth/src/auth.rs
+++ b/backend/windmill-api-auth/src/auth.rs
@@ -715,7 +715,8 @@ pub async fn resolve_opt_job_authed(
 fn username_override_from_label(label: Option<String>) -> Option<String> {
     match label {
         Some(label)
-            if label.starts_with("webhook-")
+            if label.starts_with("ephemeral-webhook-")
+                || label.starts_with("webhook-")
                 || label.starts_with("http-")
                 || label.starts_with("email-")
                 || label.starts_with("ws-") =>
diff --git a/backend/windmill-api-integration-tests/tests/token_hash.rs b/backend/windmill-api-integration-tests/tests/token_hash.rs
index 6f4ba6b0ac..45d324a3d4 100644
--- a/backend/windmill-api-integration-tests/tests/token_hash.rs
+++ b/backend/windmill-api-integration-tests/tests/token_hash.rs
@@ -274,7 +274,9 @@ async fn test_plaintext_backward_compat(db: Pool<Postgres>) -> anyhow::Result<()
     );
 
     // --- Phase 2: All workers upgraded (version >= 1.650.0) ---
-    MIN_VERSION.store(std::sync::Arc::new(MIN_VERSION_SUPPORTS_TOKEN_HASH.version().clone()));
+    MIN_VERSION.store(std::sync::Arc::new(
+        MIN_VERSION_SUPPORTS_TOKEN_HASH.version().clone(),
+    ));
 
     let resp = authed(client().post(format!("{base}/tokens/create")))
         .json(&json!({"label": "new-worker-token"}))
@@ -324,7 +326,7 @@ async fn test_plaintext_backward_compat(db: Pool<Postgres>) -> anyhow::Result<()
 async fn test_rotate_webhook_token(db: Pool<Postgres>) -> anyhow::Result<()> {
     initialize_tracing().await;
 
-    use windmill_native_triggers::{delete_token_by_hash, rotate_webhook_token};
+    use windmill_native_triggers::{delete_token_by_hash, rotate_webhook_token, ServiceName};
 
     // Insert a token directly with known values
     let original_token = "test-webhook-token-original-1234";
@@ -342,7 +344,7 @@ async fn test_rotate_webhook_token(db: Pool<Postgres>) -> anyhow::Result<()> {
     .await?;
 
     // Rotate the token
-    let rotated = rotate_webhook_token(&db, &original_hash)
+    let rotated = rotate_webhook_token(&db, &original_hash, ServiceName::Google)
         .await?
         .expect("rotate must return Some for existing token");
 
@@ -350,16 +352,27 @@ async fn test_rotate_webhook_token(db: Pool<Postgres>) -> anyhow::Result<()> {
     assert_ne!(rotated.new_token, original_token);
     assert_eq!(rotated.old_token_hash, original_hash);
 
-    // New token's hash should exist in DB
+    // New token's hash should exist in DB with the per-service label and expiration
     let new_hash = hash_token(&rotated.new_token);
-    let exists: bool = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM token WHERE token_hash = $1) AS exists",
+    let new_row = sqlx::query!(
+        "SELECT label, expiration FROM token WHERE token_hash = $1",
         new_hash
     )
-    .fetch_one(&db)
+    .fetch_optional(&db)
     .await?
-    .unwrap_or(false);
-    assert!(exists, "new token hash must exist in DB after rotation");
+    .expect("new token hash must exist in DB after rotation");
+    assert!(
+        new_row
+            .label
+            .as_deref()
+            .is_some_and(|l| l.starts_with("ephemeral-webhook-google-")),
+        "rotated token must carry an ephemeral-webhook-google-* label, got {:?}",
+        new_row.label
+    );
+    assert!(
+        new_row.expiration.is_some(),
+        "rotated Google token must carry an expiration"
+    );
 
     // Old token should still exist (deletion deferred to caller)
     let old_exists: bool = sqlx::query_scalar!(
@@ -389,7 +402,7 @@ async fn test_rotate_webhook_token(db: Pool<Postgres>) -> anyhow::Result<()> {
     assert!(!old_gone, "old token must be gone after explicit deletion");
 
     // Rotating a non-existent hash should return None
-    let result = rotate_webhook_token(&db, "nonexistent_hash").await?;
+    let result = rotate_webhook_token(&db, "nonexistent_hash", ServiceName::Google).await?;
     assert!(
         result.is_none(),
         "rotating a non-existent token must return None"
diff --git a/backend/windmill-native-triggers/src/google/external.rs b/backend/windmill-native-triggers/src/google/external.rs
index 29dd1c943f..616f8d3f63 100644
--- a/backend/windmill-native-triggers/src/google/external.rs
+++ b/backend/windmill-native-triggers/src/google/external.rs
@@ -323,9 +323,10 @@ impl Google {
     }
 
     /// Renew an expiring Google watch channel.
-    /// Rotates the webhook token (creating a new one with the same label),
-    /// stops the old channel and creates a new one with a fresh channel ID
-    /// (Google rejects reused channel IDs with `channelIdNotUnique`).
+    /// Rotates the webhook token (mints a fresh `ephemeral-webhook-google-{rd5}` label
+    /// and a 14-day expiration via `rotate_webhook_token`), stops the old channel and
+    /// creates a new one with a fresh channel ID (Google rejects reused channel IDs
+    /// with `channelIdNotUnique`).
     /// Returns (new_service_config, new_plaintext_token, old_token_hash).
     /// Callers should delete old_token_hash after successfully updating the trigger.
     pub async fn renew_channel(
@@ -341,7 +342,13 @@ impl Google {
             .transpose()?
             .ok_or_else(|| Error::InternalErr("Missing service config".to_string()))?;
 
-        let rotated = match rotate_webhook_token(db, &trigger.webhook_token_hash).await? {
+        let rotated = match rotate_webhook_token(
+            db,
+            &trigger.webhook_token_hash,
+            ServiceName::Google,
+        )
+        .await?
+        {
             Some(r) => r,
             None => {
                 return Err(Error::InternalErr(format!(
@@ -464,6 +471,115 @@ pub fn should_renew_channel(service_config: &serde_json::Value) -> bool {
     remaining_ms < renewal_window_ms
 }
 
+enum RenewOutcome {
+    Renewed,
+    /// Another replica holds the lock, or the row was already renewed.
+    Skipped,
+}
+
+/// Renew one Google watch channel under a row lock.
+/// `sync_all_triggers` runs on every replica with no leader election — without
+/// the lock, parallel renewals orphan the losers' new tokens and Google channels.
+async fn try_renew_channel_locked(
+    handler: &Google,
+    db: &DB,
+    workspace_id: &str,
+    trigger: &NativeTrigger,
+) -> Result<RenewOutcome> {
+    let mut tx = db.begin().await?;
+
+    let row = sqlx::query!(
+        r#"
+        SELECT service_config, webhook_token_hash
+        FROM native_trigger
+        WHERE workspace_id = $1
+          AND service_name = $2
+          AND external_id = $3
+        FOR UPDATE SKIP LOCKED
+        "#,
+        workspace_id,
+        ServiceName::Google as ServiceName,
+        trigger.external_id,
+    )
+    .fetch_optional(&mut *tx)
+    .await?;
+
+    let Some(row) = row else {
+        return Ok(RenewOutcome::Skipped);
+    };
+
+    let Some(service_config) = row.service_config else {
+        // Anomalous: a Google trigger row should always carry a service_config.
+        tracing::warn!(
+            "Google trigger '{}' has NULL service_config — skipping renewal",
+            trigger.external_id
+        );
+        return Ok(RenewOutcome::Skipped);
+    };
+
+    // Re-check after the lock — a contending replica may have just renewed.
+    if !should_renew_channel(&service_config) {
+        return Ok(RenewOutcome::Skipped);
+    }
+
+    // Use freshly-read fields — webhook_token_hash may have rotated since list time.
+    let fresh_trigger = NativeTrigger {
+        service_config: Some(service_config),
+        webhook_token_hash: row.webhook_token_hash,
+        ..trigger.clone()
+    };
+
+    let (new_config, new_token, old_token_hash) = handler
+        .renew_channel(workspace_id, &fresh_trigger, db)
+        .await?;
+
+    // Past this point a new Google channel exists. Any failure leaks it.
+    if let Err(e) = update_native_trigger_service_config(
+        &mut *tx,
+        workspace_id,
+        ServiceName::Google,
+        &trigger.external_id,
+        &new_config,
+        Some(&new_token),
+    )
+    .await
+    {
+        tracing::error!(
+            "DB update failed after creating new Google channel for '{}' — channel orphaned in Google: {}",
+            trigger.external_id,
+            e
+        );
+        return Err(e);
+    }
+
+    if let Err(e) = tx.commit().await {
+        tracing::error!(
+            "Commit failed after creating new Google channel for '{}' — channel orphaned in Google: {}",
+            trigger.external_id,
+            e
+        );
+        return Err(e.into());
+    }
+
+    // With the lock + rotation in place, the old token row must exist here.
+    // Ok(false) means a concurrent path deleted it (or the expiry sweep collected it).
+    match crate::delete_token_by_hash(db, &old_token_hash).await {
+        Ok(true) => {}
+        Ok(false) => tracing::warn!(
+            "Old webhook token already gone after renewal for '{}' (hash {})",
+            trigger.external_id,
+            old_token_hash
+        ),
+        Err(e) => tracing::warn!(
+            "Failed to delete old webhook token after channel renewal for '{}': {}",
+            trigger.external_id,
+            e
+        ),
+    }
+
+    Ok(RenewOutcome::Renewed)
+}
+
 async fn renew_expiring_channels(
     handler: &Google,
     db: &DB,
@@ -488,53 +604,25 @@ async fn renew_expiring_channels(
             workspace_id
         );
 
-        match handler.renew_channel(workspace_id, trigger, db).await {
-            Ok((new_config, new_token, old_token_hash)) => {
-                match update_native_trigger_service_config(
-                    db,
-                    workspace_id,
-                    ServiceName::Google,
-                    &trigger.external_id,
-                    &new_config,
-                    Some(&new_token),
-                )
-                .await
-                {
-                    Ok(()) => {
-                        // Trigger updated — clean up old token (best-effort)
-                        if let Err(e) = crate::delete_token_by_hash(db, &old_token_hash).await {
-                            tracing::warn!(
-                                "Failed to delete old webhook token after channel renewal for {}: {}",
-                                trigger.external_id, e
-                            );
-                        }
-                        tracing::info!(
-                            "Renewed Google channel {} for '{}'",
-                            trigger.external_id,
-                            trigger.script_path
-                        );
-                        synced.push(TriggerSyncInfo {
-                            external_id: trigger.external_id.clone(),
-                            script_path: trigger.script_path.clone(),
-                            action: SyncAction::ConfigUpdated,
-                        });
-                    }
-                    Err(e) => {
-                        tracing::error!(
-                            "Failed to update DB after renewing Google channel {}: {}",
-                            trigger.external_id,
-                            e
-                        );
-                        errors.push(SyncError {
-                            resource_path: format!("workspace:{}", workspace_id),
-                            error_message: format!(
-                                "Failed to update DB after channel renewal for {}: {}",
-                                trigger.external_id, e
-                            ),
-                            error_type: "channel_renewal_error".to_string(),
-                        });
-                    }
-                }
+        match try_renew_channel_locked(handler, db, workspace_id, trigger).await {
+            Ok(RenewOutcome::Renewed) => {
+                tracing::info!(
+                    "Renewed Google channel {} for '{}'",
+                    trigger.external_id,
+                    trigger.script_path
+                );
+                synced.push(TriggerSyncInfo {
+                    external_id: trigger.external_id.clone(),
+                    script_path: trigger.script_path.clone(),
+                    action: SyncAction::ConfigUpdated,
+                });
+            }
+            Ok(RenewOutcome::Skipped) => {
+                // Expected outcome under SKIP LOCKED: contending replica or already-renewed row.
+                tracing::debug!(
+                    "Skipped Google channel renewal for '{}': another replica is renewing or the row was already renewed",
+                    trigger.external_id
+                );
             }
             Err(e) => {
                 tracing::error!(
diff --git a/backend/windmill-native-triggers/src/handler.rs b/backend/windmill-native-triggers/src/handler.rs
index 91a8b9531e..1b6aa83b92 100644
--- a/backend/windmill-native-triggers/src/handler.rs
+++ b/backend/windmill-native-triggers/src/handler.rs
@@ -1,7 +1,8 @@
 use crate::{
     decrypt_oauth_data, delete_native_trigger, delete_token_by_hash, get_native_trigger,
     list_native_triggers, rotate_webhook_token, store_native_trigger, update_native_trigger_error,
-    External, NativeTrigger, NativeTriggerConfig, NativeTriggerData, ServiceName,
+    webhook_token_label, External, NativeTrigger, NativeTriggerConfig, NativeTriggerData,
+    ServiceName,
 };
 use axum::{
     extract::{Path, Query},
@@ -18,7 +19,6 @@ use windmill_audit::{audit_oss::audit_log, ActionKind};
 use windmill_common::{
     db::UserDB,
     error::{Error, JsonResult, Result},
-    utils::rd_string,
     DB,
 };
 
@@ -84,10 +84,13 @@ async fn new_webhook_token(
     let kind = if is_flow { "flows" } else { "scripts" };
 
     let scopes = vec![format!("jobs:run:{kind}:{script_path}")];
-    let label = format!("webhook-{}-{}", service_name.as_str(), rd_string(5));
+    let label = webhook_token_label(service_name);
+    let expiration = service_name
+        .webhook_token_expiration()
+        .map(|d| chrono::Utc::now() + d);
     let token_config = NewToken::new(
         Some(label),
-        None,
+        expiration,
         None,
         Some(scopes),
         Some(workspace_id.to_owned()),
@@ -255,8 +258,8 @@ async fn update_native_trigger_handler<T: External>(
         tx = user_db.begin(&authed).await?;
         token
     } else {
-        // Same runnable — rotate the token keeping the same label
-        match rotate_webhook_token(&db, &existing.webhook_token_hash).await? {
+        // Same runnable — rotate the token (mints a fresh label + expiration)
+        match rotate_webhook_token(&db, &existing.webhook_token_hash, service_name).await? {
             Some(rotated) => {
                 old_token_hash_to_delete = Some(rotated.old_token_hash);
                 rotated.new_token
diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs
index c871159b0f..743b9f8210 100644
--- a/backend/windmill-native-triggers/src/lib.rs
+++ b/backend/windmill-native-triggers/src/lib.rs
@@ -180,6 +180,17 @@ impl ServiceName {
     pub fn integration_service(&self) -> ServiceName {
         *self
     }
+
+    /// How long webhook tokens for this service should remain valid. `None` = no expiry.
+    /// Google channels turn over on a tight schedule (24h Drive, 7d Calendar) — a finite
+    /// TTL lets `delete_expired_items` (`monitor.rs`) sweep orphaned tokens automatically.
+    /// Persistent-webhook services (Nextcloud, GitHub) return `None`.
+    pub fn webhook_token_expiration(&self) -> Option<chrono::Duration> {
+        match self {
+            ServiceName::Google => Some(chrono::Duration::days(14)),
+            ServiceName::Nextcloud | ServiceName::Github => None,
+        }
+    }
 }
 
 impl std::fmt::Display for ServiceName {
@@ -759,22 +770,23 @@ async fn update_oauth_token_resource(
     }
 }
 
-/// Create a new webhook token that keeps the same label as the old one.
-/// The old token is **not** deleted — callers must call `delete_token_by_hash`
-/// on `old_token_hash` after the trigger row has been successfully updated.
-/// This ensures the trigger keeps working if the external service call or
-/// subsequent DB update fails.
+/// Create a new webhook token, minting a fresh `ephemeral-webhook-{service}-{rd5}`
+/// label and the per-service expiration (see `ServiceName::webhook_token_expiration`).
+/// The old token is **not** deleted — callers must call `delete_token_by_hash` on
+/// `old_token_hash` after the trigger row has been successfully updated.
 ///
 /// Returns `Ok(None)` if the old token no longer exists (e.g. manually deleted by user).
-/// In that case, `renew_channel` returns an error which `renew_expiring_channels` writes
-/// to the trigger's `error` column — visible in the UI so the user can re-create the trigger.
-pub async fn rotate_webhook_token(db: &DB, old_token_hash: &str) -> Result<Option<RotatedToken>> {
+pub async fn rotate_webhook_token(
+    db: &DB,
+    old_token_hash: &str,
+    service_name: ServiceName,
+) -> Result<Option<RotatedToken>> {
     use windmill_common::auth::{hash_token, TOKEN_PREFIX_LEN};
     use windmill_common::min_version::MIN_VERSION_SUPPORTS_TOKEN_HASH;
     use windmill_common::utils::rd_string;
 
     let old = match sqlx::query!(
-        "SELECT label, email, scopes, workspace_id, super_admin, owner, expiration FROM token WHERE token_hash = $1",
+        "SELECT email, scopes, workspace_id, super_admin, owner FROM token WHERE token_hash = $1",
         old_token_hash
     )
     .fetch_optional(db)
@@ -799,6 +811,11 @@ pub async fn rotate_webhook_token(db: &DB, old_token_hash: &str) -> Result<Optio
         Some(&new_token)
     };
 
+    let new_label = webhook_token_label(service_name);
+    let new_expiration = service_name
+        .webhook_token_expiration()
+        .map(|d| Utc::now() + d);
+
     sqlx::query!(
         "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes, workspace_id, owner, expiration)
          VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
@@ -806,12 +823,12 @@ pub async fn rotate_webhook_token(db: &DB, old_token_hash: &str) -> Result<Optio
         new_prefix,
         plaintext as Option<&str>,
         old.email,
-        old.label,
+        new_label,
         old.super_admin,
         old.scopes.as_deref(),
         old.workspace_id,
         old.owner,
-        old.expiration,
+        new_expiration,
     )
     .execute(db)
     .await?;
@@ -822,6 +839,19 @@ pub async fn rotate_webhook_token(db: &DB, old_token_hash: &str) -> Result<Optio
     }))
 }
 
+/// Mint the standard label used for native-trigger webhook tokens.
+/// The `ephemeral-` prefix opts the token out of the user-token email/critical-alert
+/// notification paths (`is_user_token` in `monitor.rs`, `register_token_expiry_notification`
+/// in `windmill-api-auth/src/lib.rs`, `isUserToken` in the frontend).
+pub fn webhook_token_label(service_name: ServiceName) -> String {
+    use windmill_common::utils::rd_string;
+    format!(
+        "ephemeral-webhook-{}-{}",
+        service_name.as_str(),
+        rd_string(5)
+    )
+}
+
 pub struct RotatedToken {
     pub new_token: String,
     /// Hash of the old token — callers should delete this after the
@@ -829,7 +859,10 @@ pub struct RotatedToken {
     pub old_token_hash: String,
 }
 
-/// Delete a token from the token table using its hash (exact match).
+/// Delete a token by hash. Returns `Ok(false)` when no row matched.
+/// Some call sites legitimately race against expiry sweeps or concurrent deletes;
+/// callers that consider 0-rows anomalous should log themselves at the appropriate
+/// level rather than have this helper warn unconditionally.
 pub async fn delete_token_by_hash<'c, E: sqlx::Executor<'c, Database = Postgres>>(
     db: E,
     token_hash: &str,

From 4b4aa0e303f9c47c4f931511977107f42f93abc3 Mon Sep 17 00:00:00 2001
From: hugocasa <hugo@casademont.ch>
Date: Fri, 8 May 2026 17:57:04 +0200
Subject: [PATCH 09/17] fix(cli): bump svelte version in `wmill app new`
 template (#9084)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix(cli): bump svelte version in `wmill app new` template

The svelte5 template pinned `svelte` to `5.45.2`, but the Svelte
compiler bundled in `wmill app dev` emits `$.delegated('click', ...)`
calls. The `delegated` export was added later, so 5.45.2 doesn't have
it — esbuild warns `Import "delegated" will always be undefined`,
replaces the call with `void 0`, and the page crashes at first
event-handler bind (white screen).

Bump to `^5.55.5` so the compiler and runtime stay in sync.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fix(frontend): bump svelte version in raw_apps UI template

Mirror the CLI fix: the UI's `Add raw app` flow scaffolds a
package.json with `svelte: "5.45.2"`. That works today only because
the bundled rolldown worker also pins 5.45.2 — when the worker is
upgraded past 5.51.1, the compiler will emit `$.delegated()` and the
runtime won't have it, producing the same white-page crash that hit
the CLI.

5.55.5 still exports `event` (used by the current bundled compiler),
so this is forward-compatible: it works with the 5.45.2 compiler now
and won't break when the worker is upgraded.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 cli/src/commands/app/new.ts                                   | 2 +-
 frontend/src/routes/(root)/(logged)/apps_raw/add/templates.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cli/src/commands/app/new.ts b/cli/src/commands/app/new.ts
index 2072c0312c..87d40f161b 100644
--- a/cli/src/commands/app/new.ts
+++ b/cli/src/commands/app/new.ts
@@ -168,7 +168,7 @@ const templates: Record<string, FrameworkTemplate> = {
       "/index.css": indexCss,
       "/package.json": `{
     "dependencies": {
-        "svelte": "5.45.2",
+        "svelte": "^5.55.5",
         "windmill-client": "^1"
     }
 }`,
diff --git a/frontend/src/routes/(root)/(logged)/apps_raw/add/templates.ts b/frontend/src/routes/(root)/(logged)/apps_raw/add/templates.ts
index 71225f9ce1..eaac0f108f 100644
--- a/frontend/src/routes/(root)/(logged)/apps_raw/add/templates.ts
+++ b/frontend/src/routes/(root)/(logged)/apps_raw/add/templates.ts
@@ -136,7 +136,7 @@ export const svelte5Template = {
 	'/index.css': indexCss,
 	'/package.json': `{
     "dependencies": {
-        "svelte": "5.45.2",
+        "svelte": "^5.55.5",
         "windmill-client": "^1"
     }
 }`

From 23af6c2ea31265a1898d0632e72cd2fd826e4044 Mon Sep 17 00:00:00 2001
From: hugocasa <hugo@casademont.ch>
Date: Fri, 8 May 2026 17:57:17 +0200
Subject: [PATCH 10/17] perf(flows): gate flow_env resolve on expr text and
 share cache with handle_flow (#9085)

---
 backend/windmill-worker/src/worker_flow.rs | 59 +++++++++++++++++-----
 1 file changed, 45 insertions(+), 14 deletions(-)

diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs
index 91c6cf9ec7..53c94a7458 100644
--- a/backend/windmill-worker/src/worker_flow.rs
+++ b/backend/windmill-worker/src/worker_flow.rs
@@ -494,19 +494,34 @@ pub async fn update_flow_status_after_job_completion_internal(
         }
 
         // Resolve flow_env for predicate evaluations (stop_after_if,
-        // stop_after_all_iters_if, retry_if). Only fetch when one of these
-        // predicates is configured to avoid an extra DB query on the common
-        // path. `retry` without `retry_if` doesn't consult flow_env.
-        let retry_uses_flow_env =
-            |module: &FlowModule| module.retry.as_ref().is_some_and(|r| r.retry_if.is_some());
+        // stop_after_all_iters_if, retry_if). Two-level gate:
+        //   1. Structural — at least one such predicate is configured.
+        //   2. Textual — the predicate expression actually references
+        //      `flow_env`. Mirrors the existing `expr.contains("results.")`
+        //      check in `get_id_ctx_for_expr`. A substring match has false
+        //      positives (harmless — just runs an unneeded resolve) but no
+        //      false negatives, since `flow_env` must appear textually for the
+        //      expression engine to read it.
+        let expr_uses_flow_env = |expr: &str| expr.contains("flow_env");
+        let retry_if_uses_flow_env = |module: &FlowModule| {
+            module
+                .retry
+                .as_ref()
+                .and_then(|r| r.retry_if.as_ref())
+                .is_some_and(|r| expr_uses_flow_env(&r.expr))
+        };
         let needs_flow_env = current_module.is_some_and(|m| {
-            m.stop_after_if.is_some()
-                || m.stop_after_all_iters_if.is_some()
-                || retry_uses_flow_env(m)
+            m.stop_after_if
+                .as_ref()
+                .is_some_and(|s| expr_uses_flow_env(&s.expr))
+                || m.stop_after_all_iters_if
+                    .as_ref()
+                    .is_some_and(|s| expr_uses_flow_env(&s.expr))
+                || retry_if_uses_flow_env(m)
         }) || flow_value
             .failure_module
             .as_ref()
-            .is_some_and(|fm| retry_uses_flow_env(fm));
+            .is_some_and(|fm| retry_if_uses_flow_env(fm));
         // The resolved env is constant for a flow's lifetime, so cache it by flow
         // job id and reuse across child-step completions. Cache miss falls back to
         // the existing resolve+persist path; same-worker subsequent completions are
@@ -2729,8 +2744,11 @@ pub async fn handle_flow(
     // Resolve $var: and $res: references in flow_env.
     // We resolve into a separate variable to avoid cloning the entire FlowValue
     // (which includes modules, failure_module, etc.) just to replace flow_env.
+    // `is_cacheable` is `false` only on a transient `transform_json` failure,
+    // matching `resolve_flow_env_for_status_update`'s contract — we must not
+    // freeze a partial resolution into the cache.
     let resolved_env;
-    let flow_env = if let Some(env) = env_source {
+    let (flow_env, is_cacheable) = if let Some(env) = env_source {
         match transform_json(
             client,
             &flow_job.workspace_id,
@@ -2742,18 +2760,31 @@ pub async fn handle_flow(
         {
             Ok(Some(resolved)) => {
                 resolved_env = resolved;
-                Some(&resolved_env)
+                (Some(&resolved_env), true)
             }
-            Ok(None) => Some(env),
+            Ok(None) => (Some(env), true),
             Err(e) => {
                 tracing::warn!("Failed to resolve flow_env references: {e}");
-                Some(env)
+                (Some(env), false)
             }
         }
     } else {
-        None
+        (None, true)
     };
 
+    // Populate the per-flow resolved-env cache so subsequent predicate
+    // evaluations in `update_flow_status_after_job_completion_internal` skip
+    // the recursive CTE + transform_json. Costs one HashMap clone per
+    // sub-flow handle_flow entry; saves up to one CTE + one transform_json
+    // per flow execution that has predicates referencing flow_env.
+    if is_cacheable {
+        if let Some(env) = flow_env {
+            if !env.is_empty() {
+                RESOLVED_FLOW_ENV_CACHE.insert(flow_job.id, Arc::new(env.clone()));
+            }
+        }
+    }
+
     let status = flow_job
         .parse_flow_status()
         .with_context(|| "Unable to parse flow status")?;

From dd5320205f200dd058db2ff7d44d5c4bbcf25ec9 Mon Sep 17 00:00:00 2001
From: hugocasa <hugo@casademont.ch>
Date: Fri, 8 May 2026 17:57:34 +0200
Subject: [PATCH 11/17] feat: parse windmill_failure field to tag run as
 failure (#9073)

* feat: parse windmill_failure field in job result to tag run as failure

* feat: preserve top-level fields when windmill_failure tags a run as failure

* fix: address review findings on windmill_manual_failure

* refactor: rename windmill_manual_failure to wm_failure and add wm_* aliases

* fix: prefer injected ManualFailure error over sibling name/message in OTel
---
 backend/windmill-api-jobs/src/execution.rs    |   3 +
 backend/windmill-queue/src/jobs.rs            | 156 +++++++++++++++---
 .../windmill-worker/src/result_processor.rs   | 135 ++++++++++++---
 3 files changed, 249 insertions(+), 45 deletions(-)

diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs
index 555d7a4d8b..e4359e19da 100644
--- a/backend/windmill-api-jobs/src/execution.rs
+++ b/backend/windmill-api-jobs/src/execution.rs
@@ -248,8 +248,11 @@ lazy_static::lazy_static! {
 
 #[derive(Deserialize)]
 pub struct WindmillCompositeResult {
+    #[serde(alias = "wm_status_code")]
     windmill_status_code: Option<u16>,
+    #[serde(alias = "wm_content_type")]
     windmill_content_type: Option<String>,
+    #[serde(alias = "wm_headers")]
     windmill_headers: Option<HashMap<String, String>>,
     result: Option<Box<RawValue>>,
 }
diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs
index 897f8e2ce0..52e6b09c6f 100644
--- a/backend/windmill-queue/src/jobs.rs
+++ b/backend/windmill-queue/src/jobs.rs
@@ -630,12 +630,45 @@ pub struct WrappedError {
 pub trait ValidableJson {
     fn is_valid_json(&self) -> bool;
     fn wm_labels(&self) -> Option<Vec<String>>;
+    fn wm_failure(&self) -> Option<String>;
+    fn result_metadata(&self) -> ResultMetadata;
     fn size(&self) -> usize;
 }
 
-#[derive(serde::Deserialize)]
-struct ResultLabels {
-    wm_labels: Vec<String>,
+/// The Windmill-specific markers we look for inside a job's result.
+/// `wm_failure` retags a successful run as a failure with the
+/// given message; `wm_labels` adds runtime labels to the job row.
+#[derive(serde::Deserialize, Default, Debug, Clone)]
+pub struct ResultMetadata {
+    pub wm_labels: Option<Vec<String>>,
+    pub wm_failure: Option<String>,
+}
+
+/// Sentinel `error.name` we inject into a result when retagging a successful
+/// run as a failure due to `wm_failure`. Used downstream to detect that
+/// the result is already in the standard `{ error: { name, message }, ... }`
+/// shape and must not be wrapped a second time by `WrappedError`.
+pub const MANUAL_FAILURE_ERROR_NAME: &str = "ManualFailure";
+
+/// Returns true when the result already carries our injected
+/// `error: { name: "ManualFailure", ... }` marker — i.e. it was shaped by
+/// `process_jc`'s wm_failure path. A real runtime failure whose raw
+/// result happens to contain a `wm_failure` field but no such error
+/// key returns false (and so still goes through the standard wrap path).
+pub fn is_pre_shaped_wm_failure_result(result: &str) -> bool {
+    #[derive(serde::Deserialize)]
+    struct Marker {
+        error: Option<NameOnly>,
+    }
+    #[derive(serde::Deserialize)]
+    struct NameOnly {
+        name: String,
+    }
+    serde_json::from_str::<Marker>(result)
+        .ok()
+        .and_then(|m| m.error)
+        .map(|e| e.name == MANUAL_FAILURE_ERROR_NAME)
+        .unwrap_or(false)
 }
 
 impl ValidableJson for WrappedError {
@@ -647,6 +680,14 @@ impl ValidableJson for WrappedError {
         None
     }
 
+    fn wm_failure(&self) -> Option<String> {
+        None
+    }
+
+    fn result_metadata(&self) -> ResultMetadata {
+        ResultMetadata::default()
+    }
+
     fn size(&self) -> usize {
         0
     }
@@ -658,9 +699,15 @@ impl ValidableJson for Box<RawValue> {
     }
 
     fn wm_labels(&self) -> Option<Vec<String>> {
-        serde_json::from_str::<ResultLabels>(self.get())
-            .ok()
-            .map(|r| r.wm_labels)
+        self.result_metadata().wm_labels
+    }
+
+    fn wm_failure(&self) -> Option<String> {
+        self.result_metadata().wm_failure
+    }
+
+    fn result_metadata(&self) -> ResultMetadata {
+        serde_json::from_str::<ResultMetadata>(self.get()).unwrap_or_default()
     }
 
     fn size(&self) -> usize {
@@ -677,6 +724,14 @@ impl<T: ValidableJson> ValidableJson for Arc<T> {
         T::wm_labels(&self)
     }
 
+    fn wm_failure(&self) -> Option<String> {
+        T::wm_failure(&self)
+    }
+
+    fn result_metadata(&self) -> ResultMetadata {
+        T::result_metadata(&self)
+    }
+
     fn size(&self) -> usize {
         T::size(&self)
     }
@@ -688,9 +743,15 @@ impl ValidableJson for serde_json::Value {
     }
 
     fn wm_labels(&self) -> Option<Vec<String>> {
-        serde_json::from_value::<ResultLabels>(self.clone())
-            .ok()
-            .map(|r| r.wm_labels)
+        self.result_metadata().wm_labels
+    }
+
+    fn wm_failure(&self) -> Option<String> {
+        self.result_metadata().wm_failure
+    }
+
+    fn result_metadata(&self) -> ResultMetadata {
+        serde_json::from_value::<ResultMetadata>(self.clone()).unwrap_or_default()
     }
 
     fn size(&self) -> usize {
@@ -707,6 +768,14 @@ impl<T: ValidableJson> ValidableJson for Json<T> {
         self.0.wm_labels()
     }
 
+    fn wm_failure(&self) -> Option<String> {
+        self.0.wm_failure()
+    }
+
+    fn result_metadata(&self) -> ResultMetadata {
+        self.0.result_metadata()
+    }
+
     fn size(&self) -> usize {
         self.0.size()
     }
@@ -742,16 +811,7 @@ where
     }
 }
 
-pub async fn add_completed_job_error(
-    db: &Pool<Postgres>,
-    completed_job: &MiniCompletedJob,
-    mem_peak: i32,
-    canceled_by: Option<CanceledBy>,
-    e: serde_json::Value,
-    _worker_name: &str,
-    flow_is_done: bool,
-    duration: Option<i64>,
-) -> Result<WrappedError, Error> {
+async fn record_failure_metrics(completed_job: &MiniCompletedJob, _worker_name: &str) {
     #[cfg(feature = "prometheus")]
     register_metric(
         &WORKER_EXECUTION_FAILED,
@@ -772,6 +832,64 @@ pub async fn add_completed_job_error(
     .await;
 
     otel_incr_worker_execution_failed(&completed_job.tag);
+}
+
+/// Tag a completed job as a failure while storing the result as-is, without
+/// the standard `WrappedError` `{ error: ... }` wrap. Use for jobs whose result
+/// is already shaped (e.g. when `wm_failure` injected a top-level
+/// `error` key, while preserving sibling fields like `windmill_status_code`).
+///
+/// This is a worker-internal helper called by trusted result-processing code
+/// after the worker has authenticated and pulled the job. Callers MUST verify
+/// upstream auth (i.e. the job was legitimately pulled by this worker) — this
+/// function performs no authorization check itself, mirroring the contract of
+/// `add_completed_job_error`.
+pub async fn add_completed_job_pre_shaped_failure<T: Serialize + Send + Sync + ValidableJson>(
+    db: &Pool<Postgres>,
+    completed_job: &MiniCompletedJob,
+    mem_peak: i32,
+    canceled_by: Option<CanceledBy>,
+    result: Json<&T>,
+    worker_name: &str,
+    flow_is_done: bool,
+    duration: Option<i64>,
+) -> Result<(), Error> {
+    record_failure_metrics(completed_job, worker_name).await;
+
+    tracing::error!(
+        "job {} in {} did not succeed (wm_failure)",
+        completed_job.id,
+        completed_job.workspace_id,
+    );
+    let _ = add_completed_job(
+        db,
+        completed_job,
+        false,
+        false,
+        result,
+        None,
+        mem_peak,
+        canceled_by,
+        flow_is_done,
+        duration,
+        false,
+    )
+    .warn_after_seconds(10)
+    .await?;
+    Ok(())
+}
+
+pub async fn add_completed_job_error(
+    db: &Pool<Postgres>,
+    completed_job: &MiniCompletedJob,
+    mem_peak: i32,
+    canceled_by: Option<CanceledBy>,
+    e: serde_json::Value,
+    worker_name: &str,
+    flow_is_done: bool,
+    duration: Option<i64>,
+) -> Result<WrappedError, Error> {
+    record_failure_metrics(completed_job, worker_name).await;
 
     let result = WrappedError { error: e };
     tracing::error!(
diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs
index 1f771f6879..9eac0a5dcd 100644
--- a/backend/windmill-worker/src/result_processor.rs
+++ b/backend/windmill-worker/src/result_processor.rs
@@ -35,8 +35,9 @@ use windmill_common::{
 use windmill_common::bench::{BenchmarkInfo, BenchmarkIter};
 
 use windmill_queue::{
-    append_logs, get_mini_completed_job, CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob,
-    MiniPulledJob, ValidableJson, WrappedError, INIT_SCRIPT_TAG,
+    append_logs, get_mini_completed_job, is_pre_shaped_wm_failure_result, CanceledBy, FlowRunners,
+    JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, WrappedError, INIT_SCRIPT_TAG,
+    MANUAL_FAILURE_ERROR_NAME,
 };
 
 use serde_json::{json, value::RawValue, Value};
@@ -61,8 +62,37 @@ struct ErrorMessage {
     name: String,
 }
 
+#[derive(Debug, Deserialize)]
+struct NestedErrorMessage {
+    error: ErrorMessage,
+}
+
+/// Extract `{ name, message }` from a result. Accepts both the standard
+/// top-level shape (regular runtime errors) and the nested `{ error: { name,
+/// message }, ... }` shape produced by the wm_failure injection.
+///
+/// For wm_failure-injected results, we prefer the nested error: a successful
+/// run may legitimately contain top-level `name`/`message` fields (user data
+/// named `name`/`message`), and we want OTel to record the ManualFailure
+/// rather than the user's sibling fields.
+fn extract_error_message(raw: &str) -> Option<ErrorMessage> {
+    let nested = serde_json::from_str::<NestedErrorMessage>(raw)
+        .ok()
+        .map(|n| n.error);
+    if matches!(&nested, Some(em) if em.name == MANUAL_FAILURE_ERROR_NAME) {
+        return nested;
+    }
+    if let Ok(em) = serde_json::from_str::<ErrorMessage>(raw) {
+        return Some(em);
+    }
+    nested
+}
+
+/// Returns the post-processing `success` value (after any `wm_failure`
+/// override). Callers use this to make worker-loop decisions that depend on
+/// whether the job ultimately succeeded — e.g. the init-script killpill.
 async fn process_jc(
-    jc: JobCompleted,
+    mut jc: JobCompleted,
     worker_name: &str,
     base_internal_url: &str,
     db: &DB,
@@ -73,7 +103,32 @@ async fn process_jc(
     killpill_rx: &tokio::sync::broadcast::Receiver<()>,
     #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
     #[cfg(feature = "benchmark")] bench_infos: &mut BenchmarkInfo,
-) {
+) -> bool {
+    // Parse `wm_labels` and `wm_failure` together (single `from_str`)
+    // so we don't deserialize the whole result twice on every job.
+    let metadata = jc.result.result_metadata();
+
+    // If the script returned a `wm_failure: <string>` field in its
+    // result, tag the run as a failure. Inject an `error: { name, message }`
+    // at the top level so error handlers / UI / OTel see the standard error
+    // shape, while preserving sibling fields (`windmill_status_code`,
+    // `windmill_content_type`, `windmill_headers`, the user's data) at the
+    // top level so sync webhook responses still honor them.
+    if jc.success {
+        if let Some(failure_msg) = metadata.wm_failure.as_ref() {
+            if let Ok(Value::Object(mut map)) = serde_json::from_str::<Value>(jc.result.get()) {
+                map.insert(
+                    "error".to_string(),
+                    json!({ "name": MANUAL_FAILURE_ERROR_NAME, "message": failure_msg }),
+                );
+                if let Ok(raw) = serde_json::value::to_raw_value(&Value::Object(map)) {
+                    jc.result = Arc::new(raw);
+                }
+            }
+            jc.success = false;
+        }
+    }
+
     let success: bool = jc.success;
 
     let span = if success {
@@ -125,7 +180,7 @@ async fn process_jc(
         jc.job.id
     };
 
-    if let Some(labels) = jc.result.wm_labels() {
+    if let Some(labels) = metadata.wm_labels.as_ref() {
         if !labels.is_empty() {
             span.record("labels", labels.join(","));
         }
@@ -163,7 +218,7 @@ async fn process_jc(
         span.record("script_hash", script_hash.to_string().as_str());
     }
     if !success {
-        if let Ok(result_error) = serde_json::from_str::<ErrorMessage>(jc.result.get()) {
+        if let Some(result_error) = extract_error_message(jc.result.get()) {
             span.record("error.message", result_error.message.as_str());
             span.record("error.name", result_error.name.as_str());
             span.record(
@@ -218,6 +273,8 @@ async fn process_jc(
         )
         .await;
     }
+
+    success
 }
 
 enum JobCompletedRx {
@@ -311,8 +368,7 @@ pub fn start_background_processor(
                     result: SendResultPayload::JobCompleted(jc),
                     time,
                 }) => {
-                    let is_init_script_and_failure =
-                        !jc.success && jc.job.tag.as_str() == INIT_SCRIPT_TAG;
+                    let is_init_script = jc.job.tag.as_str() == INIT_SCRIPT_TAG;
                     let is_dependency_job = matches!(
                         jc.job.kind,
                         JobKind::Dependencies | JobKind::FlowDependencies
@@ -322,7 +378,10 @@ pub fn start_background_processor(
                     #[cfg(feature = "benchmark")]
                     let is_top_level_job = jc.job.parent_job.is_none();
 
-                    process_jc(
+                    // process_jc returns the post-override success value so a
+                    // job that flipped to failure via `wm_failure` still
+                    // triggers the init-script killpill.
+                    let final_success = process_jc(
                         jc,
                         &worker_name,
                         &base_internal_url,
@@ -340,7 +399,7 @@ pub fn start_background_processor(
                     .warn_after_seconds(10)
                     .await;
 
-                    if is_init_script_and_failure {
+                    if is_init_script && !final_success {
                         tracing::error!("init script errored, exiting");
                         killpill_tx.send();
                         break;
@@ -793,19 +852,44 @@ pub async fn process_completed_job(
             }
         }
     } else {
-        let result = add_completed_job_error(
-            db,
-            &job,
-            mem_peak.to_owned(),
-            canceled_by.clone(),
-            serde_json::from_str(result.get()).unwrap_or_else(
-                |_| json!({ "message": format!("Non serializable error: {}", result.get()) }),
-            ),
-            worker_name,
-            false,
-            None,
-        )
-        .await?;
+        // The result already carries our injected
+        // `error: { name: "ManualFailure", ... }` marker when process_jc
+        // retagged a successful run as a failure — store it as-is to preserve
+        // sibling fields like `windmill_status_code`. We check for the
+        // injected marker specifically (not just the presence of a
+        // `wm_failure` field) so a real runtime failure whose raw
+        // result happens to contain a `wm_failure` field still goes
+        // through the standard `WrappedError { error: ... }` wrap path.
+        let downstream_result: Arc<Box<RawValue>> = if is_pre_shaped_wm_failure_result(result.get())
+        {
+            windmill_queue::add_completed_job_pre_shaped_failure(
+                db,
+                &job,
+                mem_peak.to_owned(),
+                canceled_by.clone(),
+                Json(&*result),
+                worker_name,
+                false,
+                None,
+            )
+            .await?;
+            result.clone()
+        } else {
+            let wrapped = add_completed_job_error(
+                db,
+                &job,
+                mem_peak.to_owned(),
+                canceled_by.clone(),
+                serde_json::from_str(result.get()).unwrap_or_else(
+                    |_| json!({ "message": format!("Non serializable error: {}", result.get()) }),
+                ),
+                worker_name,
+                false,
+                None,
+            )
+            .await?;
+            Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap())
+        };
         if job.is_flow_step() {
             if let Some(parent_job) = job.parent_job {
                 tracing::error!(parent_flow = %parent_job, subflow = %job.id, "process completed job error, updating flow status");
@@ -817,7 +901,7 @@ pub async fn process_completed_job(
                     &job.workspace_id,
                     false,
                     canceled_by,
-                    Arc::new(serde_json::value::to_raw_value(&result).unwrap()),
+                    downstream_result,
                     duration.and_then(|d| {
                         job.started_at.map(|started_at| FlowJobDuration {
                             started_at: started_at,
@@ -855,13 +939,12 @@ pub async fn process_completed_job(
             .fetch_optional(db)
             .await?;
             if let Some(Some(job_ids)) = job_ids_json {
-                let err_result = Arc::new(serde_json::value::to_raw_value(&result).unwrap());
                 if let Ok(Some(_)) = handle_wac_child_completion(
                     db,
                     &job.id,
                     parent_job,
                     &job.workspace_id,
-                    err_result,
+                    downstream_result,
                     false,
                     job_ids,
                 )

From 935c666d50ef30d89a3669c76094af7506fbb448 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Fri, 8 May 2026 18:56:42 +0200
Subject: [PATCH 12/17] fix: hide _ENTRYPOINT_OVERRIDE jobs from script/flow
 history panel (#9088)

---
 backend/windmill-api-inputs/src/lib.rs                 |  1 +
 backend/windmill-api-jobs/src/concurrency_groups.rs    |  1 +
 backend/windmill-api-jobs/src/query.rs                 | 10 ++++++++++
 backend/windmill-api-jobs/src/types.rs                 |  5 +++++
 backend/windmill-api/openapi.yaml                      |  5 +++++
 frontend/src/lib/components/HistoricInputs.svelte      |  3 ++-
 .../src/lib/components/runs/useJobsLoader.svelte.ts    |  5 ++++-
 7 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/backend/windmill-api-inputs/src/lib.rs b/backend/windmill-api-inputs/src/lib.rs
index 154e02e5a9..348b434eea 100644
--- a/backend/windmill-api-inputs/src/lib.rs
+++ b/backend/windmill-api-inputs/src/lib.rs
@@ -172,6 +172,7 @@ async fn get_input_history(
             kind IN ('preview', 'flowpreview') as is_preview \
             FROM v2_job JOIN v2_job_completed USING (id) \
             WHERE v2_job.workspace_id = $3 AND {} = $1 AND kind = any($2) \
+            AND v2_job.script_entrypoint_override IS NULL \
             {args_query} AND v2_job_completed.status != 'skipped' {include_non_root} \
             ORDER BY v2_job.created_at DESC LIMIT $4\
         ) t ORDER BY completed_at DESC LIMIT $5 OFFSET $6",
diff --git a/backend/windmill-api-jobs/src/concurrency_groups.rs b/backend/windmill-api-jobs/src/concurrency_groups.rs
index f3ef3b14a8..d8e7a54112 100644
--- a/backend/windmill-api-jobs/src/concurrency_groups.rs
+++ b/backend/windmill-api-jobs/src/concurrency_groups.rs
@@ -226,6 +226,7 @@ async fn get_concurrent_intervals(
             trigger_kind: _,
             include_args: _,
             broad_filter: _,
+            excludes_entrypoint_override: _,
         } => true,
         _ => false,
     };
diff --git a/backend/windmill-api-jobs/src/query.rs b/backend/windmill-api-jobs/src/query.rs
index 38e139b77b..e6c57e6c5b 100644
--- a/backend/windmill-api-jobs/src/query.rs
+++ b/backend/windmill-api-jobs/src/query.rs
@@ -216,6 +216,10 @@ pub fn filter_list_queue_query(
         sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'");
     }
 
+    if lq.excludes_entrypoint_override.unwrap_or(false) {
+        sqlb.and_where_is_null("v2_job.script_entrypoint_override");
+    }
+
     if let Some(tk) = &lq.trigger_kind {
         let quoted: Vec<_> = tk.values.iter().map(|v| quote(&format!("{}", v))).collect();
         if tk.negated {
@@ -519,6 +523,10 @@ pub fn filter_list_completed_query(
         sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'");
     }
 
+    if lq.excludes_entrypoint_override.unwrap_or(false) {
+        sqlb.and_where_is_null("v2_job.script_entrypoint_override");
+    }
+
     if let Some(tk) = &lq.trigger_kind {
         let quoted: Vec<_> = tk.values.iter().map(|v| quote(&format!("{}", v))).collect();
         if tk.negated {
@@ -624,6 +632,7 @@ mod tests {
             trigger_path: None,
             include_args: None,
             broad_filter: None,
+            excludes_entrypoint_override: None,
         }
     }
 
@@ -668,6 +677,7 @@ mod tests {
             trigger_path: None,
             include_args: None,
             broad_filter: None,
+            excludes_entrypoint_override: None,
         }
     }
 
diff --git a/backend/windmill-api-jobs/src/types.rs b/backend/windmill-api-jobs/src/types.rs
index e07328e2be..bc03a04289 100644
--- a/backend/windmill-api-jobs/src/types.rs
+++ b/backend/windmill-api-jobs/src/types.rs
@@ -122,6 +122,7 @@ pub struct ListQueueQuery {
     pub trigger_path: Option<NegatedListFilter<String>>,
     pub include_args: Option<bool>,
     pub broad_filter: Option<String>,
+    pub excludes_entrypoint_override: Option<bool>,
 }
 
 #[derive(Deserialize, Clone)]
@@ -167,6 +168,7 @@ pub struct ListCompletedQuery {
     pub trigger_path: Option<NegatedListFilter<String>>,
     pub include_args: Option<bool>,
     pub broad_filter: Option<String>,
+    pub excludes_entrypoint_override: Option<bool>,
 }
 
 impl From<ListCompletedQuery> for ListQueueQuery {
@@ -202,6 +204,7 @@ impl From<ListCompletedQuery> for ListQueueQuery {
             trigger_path: lcq.trigger_path,
             include_args: lcq.include_args,
             broad_filter: lcq.broad_filter,
+            excludes_entrypoint_override: lcq.excludes_entrypoint_override,
         }
     }
 }
@@ -704,6 +707,7 @@ mod tests {
             trigger_path: None,
             include_args: None,
             broad_filter: None,
+            excludes_entrypoint_override: None,
         };
 
         let lqq: ListQueueQuery = lcq.into();
@@ -772,6 +776,7 @@ mod tests {
             trigger_path: None,
             include_args: None,
             broad_filter: None,
+            excludes_entrypoint_override: None,
         };
 
         let lqq: ListQueueQuery = lcq.into();
diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml
index 0c8d63298e..7cb96a3e29 100644
--- a/backend/windmill-api/openapi.yaml
+++ b/backend/windmill-api/openapi.yaml
@@ -12000,6 +12000,11 @@ paths:
           in: query
           schema:
             type: boolean
+        - name: excludes_entrypoint_override
+          description: exclude jobs that were started with a `_ENTRYPOINT_OVERRIDE` arg (e.g. dynamic-select helper runs and preprocessor previews)
+          in: query
+          schema:
+            type: boolean
         - name: broad_filter
           description: broad search across multiple fields (case-insensitive substring match on path, tag, schedule path, trigger kind, label)
           in: query
diff --git a/frontend/src/lib/components/HistoricInputs.svelte b/frontend/src/lib/components/HistoricInputs.svelte
index ca01013e0a..3c784f354f 100644
--- a/frontend/src/lib/components/HistoricInputs.svelte
+++ b/frontend/src/lib/components/HistoricInputs.svelte
@@ -112,7 +112,8 @@
 				syncQueuedRunsCount: false,
 				refreshRate: 10000,
 				currentWorkspace: $workspaceStore ?? '',
-				skip: !runnableId
+				skip: !runnableId,
+				excludesEntrypointOverride: true
 			}) satisfies UseJobLoaderArgs
 	)
 	let jobs = $derived(jobsLoader?.jobs ?? [])
diff --git a/frontend/src/lib/components/runs/useJobsLoader.svelte.ts b/frontend/src/lib/components/runs/useJobsLoader.svelte.ts
index b75355a6cb..e7d8337708 100644
--- a/frontend/src/lib/components/runs/useJobsLoader.svelte.ts
+++ b/frontend/src/lib/components/runs/useJobsLoader.svelte.ts
@@ -53,6 +53,7 @@ export interface UseJobLoaderArgs {
 	skip?: boolean
 	lookback?: number
 	perPage?: number
+	excludesEntrypointOverride?: boolean
 }
 
 export function useJobsLoader(args: () => UseJobLoaderArgs) {
@@ -69,6 +70,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) {
 	let lookback = $derived(_args.lookback ?? 0)
 	let timeframe = $derived(_args?.timeframe)
 	let perPage = $derived(_args?.perPage ?? 1000)
+	let excludesEntrypointOverride = $derived(_args.excludesEntrypointOverride ?? false)
 
 	let label = $derived(filters?.label ?? null)
 	let worker = $derived(filters?.worker ?? null)
@@ -274,7 +276,8 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) {
 			allWorkspaces: allWorkspaces ? true : undefined,
 			perPage: perPageOverride ?? perPage,
 			allowWildcards: allowWildcards ? true : undefined,
-			broadFilter
+			broadFilter,
+			excludesEntrypointOverride: excludesEntrypointOverride ? true : undefined
 		})
 		promise = CancelablePromiseUtils.catchErr(promise, (e) => {
 			if (e instanceof CancelError) return CancelablePromiseUtils.err(e)

From f37d3606446d23f8b11a94ea1ce5f5d4836fae17 Mon Sep 17 00:00:00 2001
From: Arnaud <31803803+Araden14@users.noreply.github.com>
Date: Fri, 8 May 2026 18:57:36 +0200
Subject: [PATCH 13/17] fix(flows): populate error handler input args from
 failure picker (#9087)

* fix(flows): populate error handler input args from failure picker

* style(flows): fix indentation in failure-step branch
---
 .../lib/components/flows/stepsInputArgs.svelte.ts   | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/frontend/src/lib/components/flows/stepsInputArgs.svelte.ts b/frontend/src/lib/components/flows/stepsInputArgs.svelte.ts
index 025bca1bfe..06a4972ee5 100644
--- a/frontend/src/lib/components/flows/stepsInputArgs.svelte.ts
+++ b/frontend/src/lib/components/flows/stepsInputArgs.svelte.ts
@@ -4,10 +4,10 @@ import {
 	dfs,
 	getPreviousModule,
 	getStepPropPicker,
-	type PickableProperties
+	type PickableProperties,
+	getFailureStepPropPicker
 } from './previousResults'
 import { evalValue } from './utils.svelte'
-
 export class StepsInputArgs {
 	#stepsEvaluated = $state<Record<string, Record<string, any>>>({})
 	#steps = $state<Record<string, Record<string, any>>>({})
@@ -158,6 +158,15 @@ export class StepsInputArgs {
 		flow: OpenFlow | undefined,
 		previewArgs: Record<string, any> | undefined
 	) {
+		if (id === 'failure' && flow && flow.value.failure_module && flowState) {
+			const picker = getFailureStepPropPicker(flowState, flow, previewArgs)
+			this.initializeFromSchema(
+				flow.value.failure_module,
+				flowState['failure']?.schema ?? {},
+				picker.pickableProperties
+			)
+			return
+		}
 		if (!flowState || !flow) {
 			return
 		}

From 98ff146cfabf45418c95c027ad6d07b08069cfcd Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Fri, 8 May 2026 19:56:51 +0200
Subject: [PATCH 14/17] fix(python): verify wheel RECORD on cache pull/install,
 finalize piptar (#9090)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Python per-package dependency cache could persist an incomplete wheel
extraction with `.valid.windmill` set, then propagate that broken artifact
to every worker through the object store. Customer hit this on
argon2-cffi==25.1.0 (missing argon2/_utils.py), and previously on
botocore/httpx (truncated tars). Symptom is a runtime ImportError that
looks like a missing dependency declaration rather than a Windmill bug.

Three changes that together stop the propagation:

1. After `pull_from_tar`, parse the wheel's `<dist-info>/RECORD` and
   confirm every listed path exists on disk before writing
   `.valid.windmill`. On failure, wipe the directory and fall through
   to a fresh local install — the next install also self-heals the
   broken object-store entry by pushing a fresh tar.

2. After `uv pip install` succeeds, run the same RECORD check before
   queuing the piptar upload or writing `.valid.windmill`. A bad install
   never becomes the source of a broken tar in the object store.

3. Finalize the tar (`drop(tar.into_inner()?)`) before reading its bytes
   for upload, so we never push an unfinalized archive (no end-of-archive
   marker) to the object store.

Verified with a 60-package end-to-end integration test (first-fill →
clear-local-cache → re-pull-from-objectstore → corrupt-objectstore-tar
→ detect-and-self-heal). All 27 packages on the live test pulled cleanly,
and the deliberately corrupted argon2-cffi tar was caught with the exact
expected log line ("wheel RECORD lists files missing on disk: argon2/_utils.py")
and replaced with a fresh tar.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 backend/windmill-worker/src/global_cache.rs   |   4 +
 .../windmill-worker/src/python_executor.rs    | 135 ++++++++++++++++++
 2 files changed, 139 insertions(+)

diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs
index 23375746fd..67a8484b82 100644
--- a/backend/windmill-worker/src/global_cache.rs
+++ b/backend/windmill-worker/src/global_cache.rs
@@ -40,6 +40,9 @@ pub async fn build_tar_and_push(
     let tar_file = std::fs::File::create(&tar_path)?;
     let mut tar = tar::Builder::new(tar_file);
     tar.append_dir_all(".", &folder)?;
+    // Write the trailing zero blocks and close the inner file BEFORE std::fs::read
+    // below. Without this, the bytes we upload to S3 are an unfinalized archive.
+    drop(tar.into_inner()?);
 
     let tar_metadata = tokio::fs::metadata(&tar_path).await;
     if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
@@ -231,6 +234,7 @@ pub async fn save_cache(
             let tar_file = std::fs::File::create(&tar_path)?;
             let mut tar = tar::Builder::new(tar_file);
             tar.append_dir_all(".", &origin)?;
+            drop(tar.into_inner()?);
             let tar_metadata = tokio::fs::metadata(&tar_path).await;
             if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
                 tracing::info!("Failed to tar cache: {origin}");
diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs
index 5fb1278976..4db7b20fd5 100644
--- a/backend/windmill-worker/src/python_executor.rs
+++ b/backend/windmill-worker/src/python_executor.rs
@@ -2113,6 +2113,84 @@ async fn spawn_uv_install(
     }
 }
 
+/// Verify that every file listed in the wheel's RECORD exists on disk under
+/// `venv_p`. Used as a structural integrity check after both a successful
+/// `pull_from_tar` (object-store cache hit) and a successful local
+/// `uv pip install`, so a truncated tar or a dropped wheel entry can never
+/// become an authoritative cache entry. Returns Err with a short description
+/// on the first integrity issue (no .dist-info, no RECORD, or any listed
+/// path missing on disk).
+async fn verify_wheel_record(venv_p: &str) -> Result<(), String> {
+    let mut entries = tokio::fs::read_dir(venv_p)
+        .await
+        .map_err(|e| format!("read_dir({venv_p}): {e}"))?;
+
+    let mut dist_info: Option<String> = None;
+    loop {
+        match entries.next_entry().await {
+            Ok(Some(entry)) => {
+                let name = entry.file_name();
+                let name_s = name.to_string_lossy();
+                if name_s.ends_with(".dist-info") {
+                    if let Ok(ft) = entry.file_type().await {
+                        if ft.is_dir() {
+                            dist_info = Some(name_s.into_owned());
+                            break;
+                        }
+                    }
+                }
+            }
+            Ok(None) => break,
+            Err(e) => return Err(format!("read_dir entry in {venv_p}: {e}")),
+        }
+    }
+
+    let dist_info = match dist_info {
+        Some(d) => d,
+        None => return Err(format!("no .dist-info directory in {venv_p}")),
+    };
+
+    let record_path = format!("{venv_p}/{dist_info}/RECORD");
+    let record_content = tokio::fs::read_to_string(&record_path)
+        .await
+        .map_err(|e| format!("read RECORD at {record_path}: {e}"))?;
+
+    let mut missing: Vec<String> = Vec::new();
+    for line in record_content.lines() {
+        let trimmed = line.trim();
+        if trimmed.is_empty() {
+            continue;
+        }
+        let rel_path = match trimmed.split(',').next() {
+            Some(p) if !p.is_empty() => p,
+            _ => continue,
+        };
+        // Defensive: skip absolute paths or escaping entries — we only
+        // validate package-relative files.
+        if rel_path.starts_with('/') || rel_path.contains("..") {
+            continue;
+        }
+        let full = format!("{venv_p}/{rel_path}");
+        if tokio::fs::metadata(&full).await.is_err() {
+            missing.push(rel_path.to_string());
+            // Bound error size in pathological cases (e.g. wholly empty dir).
+            if missing.len() >= 10 {
+                missing.push("...".to_string());
+                break;
+            }
+        }
+    }
+
+    if missing.is_empty() {
+        Ok(())
+    } else {
+        Err(format!(
+            "wheel RECORD lists files missing on disk: {}",
+            missing.join(", ")
+        ))
+    }
+}
+
 /// uv pip install, include cached or pull from S3
 pub async fn handle_python_reqs(
     requirements: Vec<String>,
@@ -2484,6 +2562,30 @@ pub async fn handle_python_reqs(
                                     workspace_id = %w_id,
                                     "No tarball was found for {venv_p} on S3 or different problem occurred {job_id}:\n{e}",
                                 );
+                            } else if let Err(verify_err) = verify_wheel_record(&venv_p).await {
+                                // The object-store tar extracted cleanly but the resulting
+                                // directory is missing files referenced by the wheel RECORD.
+                                // Wipe the broken cache entry and fall through to a fresh
+                                // local install rather than treating it as authoritative.
+                                tracing::warn!(
+                                    workspace_id = %w_id,
+                                    job_id = %job_id,
+                                    "Object-store cache for {venv_p} failed wheel RECORD verification, will reinstall locally: {verify_err}"
+                                );
+                                if let Err(rm_err) = tokio::fs::remove_dir_all(&venv_p).await {
+                                    tracing::warn!(
+                                        workspace_id = %w_id,
+                                        "could not remove broken cache dir {venv_p}: {rm_err}"
+                                    );
+                                }
+                                append_logs(
+                                    &job_id,
+                                    &w_id,
+                                    format!(
+                                        "\n[!] cached wheel for {req} from object store failed integrity check, reinstalling: {verify_err}\n"
+                                    ),
+                                    &conn,
+                                ).await;
                             } else {
                                 print_success(
                                     true,
@@ -2640,6 +2742,39 @@ pub async fn handle_python_reqs(
                 let _ = std::fs::remove_file(format!("{job_dir}/{req}.config.proto"));
             }
 
+            // Verify the install before declaring success: if uv exited 0 but
+            // the on-disk directory is missing files the wheel RECORD says
+            // should exist, do NOT write .valid.windmill, do NOT queue the
+            // piptar upload, and fail the job. This prevents a broken tar
+            // from ever being pushed to the object store and propagated to
+            // every other replica.
+            if let Err(verify_err) = verify_wheel_record(&venv_p).await {
+                tracing::error!(
+                    workspace_id = %w_id,
+                    job_id = %job_id,
+                    "uv pip install of {req} into {venv_p} failed wheel RECORD verification: {verify_err}"
+                );
+                append_logs(
+                    &job_id,
+                    &w_id,
+                    format!(
+                        "\nWheel RECORD verification failed after install of {req}: {verify_err}. \
+                         Aborting to avoid publishing a corrupt cache entry."
+                    ),
+                    &conn,
+                ).await;
+                if let Err(rm_err) = tokio::fs::remove_dir_all(&venv_p).await {
+                    tracing::warn!(
+                        workspace_id = %w_id,
+                        "could not remove broken install dir {venv_p}: {rm_err}"
+                    );
+                }
+                pids.lock().await.get_mut(i).and_then(|e| e.take());
+                return Err(Error::from(anyhow!(
+                    "wheel RECORD verification failed after install of {req}"
+                )));
+            }
+
             print_success(
                 false,
                 s3_push,

From 91ddb930c3360d5c06156f9f14da60cebd1949e7 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Fri, 8 May 2026 20:03:06 +0200
Subject: [PATCH 15/17] chore(main): release 1.699.0 (#9082)

* chore(main): release 1.699.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
---
 CHANGELOG.md                                  |  22 +++
 backend/Cargo.lock                            | 160 +++++++++---------
 backend/Cargo.toml                            |   4 +-
 .../parsers/windmill-parser-wasm/Cargo.lock   |  48 +++---
 .../parsers/windmill-parser-wasm/Cargo.toml   |   2 +-
 backend/windmill-api/openapi.yaml             |   2 +-
 benchmarks/lib.ts                             |   2 +-
 cli/src/main.ts                               |   2 +-
 frontend/package-lock.json                    |   4 +-
 frontend/package.json                         |   2 +-
 lsp/Pipfile                                   |   2 +-
 openflow.openapi.yaml                         |   2 +-
 .../WindmillClient/WindmillClient.psd1        |   2 +-
 python-client/wmill/pyproject.toml            |   2 +-
 typescript-client/jsr.json                    |   2 +-
 typescript-client/package.json                |   2 +-
 version.txt                                   |   2 +-
 17 files changed, 142 insertions(+), 120 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1034bc4f87..98cdc3b50a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,27 @@
 # Changelog
 
+## [1.699.0](https://github.com/windmill-labs/windmill/compare/v1.698.0...v1.699.0) (2026-05-08)
+
+
+### Features
+
+* parse windmill_failure field to tag run as failure ([#9073](https://github.com/windmill-labs/windmill/issues/9073)) ([dd53202](https://github.com/windmill-labs/windmill/commit/dd5320205f200dd058db2ff7d44d5c4bbcf25ec9))
+
+
+### Bug Fixes
+
+* **cli:** bump svelte version in `wmill app new` template ([#9084](https://github.com/windmill-labs/windmill/issues/9084)) ([4b4aa0e](https://github.com/windmill-labs/windmill/commit/4b4aa0e303f9c47c4f931511977107f42f93abc3))
+* **flows:** populate error handler input args from failure picker ([#9087](https://github.com/windmill-labs/windmill/issues/9087)) ([f37d360](https://github.com/windmill-labs/windmill/commit/f37d3606446d23f8b11a94ea1ce5f5d4836fae17))
+* hide _ENTRYPOINT_OVERRIDE jobs from script/flow history panel ([#9088](https://github.com/windmill-labs/windmill/issues/9088)) ([935c666](https://github.com/windmill-labs/windmill/commit/935c666d50ef30d89a3669c76094af7506fbb448))
+* **native-triggers:** serialize Google channel renewal across replicas ([#9060](https://github.com/windmill-labs/windmill/issues/9060)) ([ee3d82f](https://github.com/windmill-labs/windmill/commit/ee3d82f01f52d835218f544dad6de9b7c3184fbb))
+* **python:** verify wheel RECORD on cache pull/install, finalize piptar ([#9090](https://github.com/windmill-labs/windmill/issues/9090)) ([98ff146](https://github.com/windmill-labs/windmill/commit/98ff146cfabf45418c95c027ad6d07b08069cfcd))
+* reject root-rooted paths in ansible playbook validator on windows ([#9081](https://github.com/windmill-labs/windmill/issues/9081)) ([d37277d](https://github.com/windmill-labs/windmill/commit/d37277d2341c83faf72efa0035cbf70e2cfbd596))
+
+
+### Performance Improvements
+
+* **flows:** gate flow_env resolve on expr text and share cache with handle_flow ([#9085](https://github.com/windmill-labs/windmill/issues/9085)) ([23af6c2](https://github.com/windmill-labs/windmill/commit/23af6c2ea31265a1898d0632e72cd2fd826e4044))
+
 ## [1.698.0](https://github.com/windmill-labs/windmill/compare/v1.697.0...v1.698.0) (2026-05-08)
 
 
diff --git a/backend/Cargo.lock b/backend/Cargo.lock
index 47aff74297..6952682391 100644
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -2213,9 +2213,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.61"
+version = "1.2.62"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d"
+checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -16025,7 +16025,7 @@ dependencies = [
 
 [[package]]
 name = "windmill"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -16106,7 +16106,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-ai"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "async-trait",
  "aws-config",
@@ -16130,7 +16130,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-alerting"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16143,7 +16143,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "argon2",
@@ -16286,7 +16286,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-agent-workers"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16309,7 +16309,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-assets"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16322,7 +16322,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-auth"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16348,7 +16348,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-client"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "reqwest 0.12.28",
  "serde",
@@ -16358,7 +16358,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-configs"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16375,7 +16375,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-debug"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "base64 0.22.1",
@@ -16397,7 +16397,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-embeddings"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16420,7 +16420,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-flow-conversations"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16436,7 +16436,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-flows"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16457,7 +16457,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-groups"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16478,7 +16478,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-inputs"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16492,7 +16492,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-integration-tests"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -16524,7 +16524,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-jobs"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16549,7 +16549,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-npm-proxy"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "flate2",
@@ -16567,7 +16567,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-openapi"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16589,7 +16589,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-schedule"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16609,7 +16609,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-scripts"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16639,7 +16639,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-settings"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16667,7 +16667,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-sse"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "lazy_static",
  "serde",
@@ -16679,7 +16679,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-users"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "argon2",
  "axum 0.8.4",
@@ -16704,7 +16704,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-workers"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16718,7 +16718,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-workspaces"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "axum 0.8.4",
  "chrono",
@@ -16751,7 +16751,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-audit"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "chrono",
  "lazy_static",
@@ -16765,7 +16765,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-autoscaling"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "axum 0.8.4",
@@ -16784,7 +16784,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-common"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "aes-gcm",
  "aho-corasick",
@@ -16885,7 +16885,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-dep-map"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "chrono",
  "itertools 0.14.0",
@@ -16904,7 +16904,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-git-sync"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "regex",
  "serde",
@@ -16919,7 +16919,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-indexer"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "astral-tokio-tar",
@@ -16943,7 +16943,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-jseval"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "futures",
@@ -16960,7 +16960,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-macros"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "itertools 0.14.0",
  "lazy_static",
@@ -16976,7 +16976,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-mcp"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16997,7 +16997,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-native-triggers"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17028,7 +17028,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-oauth"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -17053,7 +17053,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-object-store"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -17087,7 +17087,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-operator"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "futures",
@@ -17105,7 +17105,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "convert_case 0.6.0",
  "serde",
@@ -17114,7 +17114,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -17126,7 +17126,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-csharp"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -17138,7 +17138,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "gosyn",
@@ -17150,7 +17150,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-graphql"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -17162,7 +17162,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-java"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -17174,7 +17174,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-nu"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "nu-parser",
@@ -17185,7 +17185,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-php"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -17196,7 +17196,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -17208,7 +17208,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-asset"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -17219,7 +17219,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -17241,7 +17241,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-r"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -17253,7 +17253,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ruby"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -17267,7 +17267,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-rust"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "convert_case 0.6.0",
@@ -17284,7 +17284,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -17297,7 +17297,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql-asset"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde",
@@ -17309,7 +17309,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -17327,7 +17327,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts-asset"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde-wasm-bindgen",
@@ -17343,7 +17343,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wac"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -17359,7 +17359,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-yaml"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde",
@@ -17370,7 +17370,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-queue"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -17407,7 +17407,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-runtime-nativets"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "const_format",
@@ -17445,7 +17445,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-sql-datatype-parser-wasm"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "getrandom 0.3.4",
  "wasm-bindgen",
@@ -17456,7 +17456,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-store"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -17486,7 +17486,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-test-utils"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17510,7 +17510,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17543,7 +17543,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-azure"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17576,7 +17576,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-email"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17596,7 +17596,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-gcp"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17630,7 +17630,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-http"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17666,7 +17666,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-kafka"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17689,7 +17689,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-mqtt"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17713,7 +17713,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-nats"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -17737,7 +17737,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-postgres"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17772,7 +17772,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-sqs"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17800,7 +17800,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-websocket"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -17823,7 +17823,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-types"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "bitflags 2.9.4",
@@ -17842,7 +17842,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-once-cell",
@@ -17955,7 +17955,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker-volumes"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "bytes",
  "futures",
diff --git a/backend/Cargo.toml b/backend/Cargo.toml
index 1a1b1f09d0..042179683d 100644
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.698.0"
+version = "1.699.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -87,7 +87,7 @@ members = [
 exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
 
 [workspace.package]
-version = "1.698.0"
+version = "1.699.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock
index 0bc102cc48..d337f4838d 100644
--- a/backend/parsers/windmill-parser-wasm/Cargo.lock
+++ b/backend/parsers/windmill-parser-wasm/Cargo.lock
@@ -6183,7 +6183,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windmill-common"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "aho-corasick",
  "anyhow",
@@ -6263,7 +6263,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-macros"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6275,7 +6275,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "convert_case",
  "serde",
@@ -6284,7 +6284,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6296,7 +6296,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-csharp"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -6308,7 +6308,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "gosyn",
@@ -6320,7 +6320,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-graphql"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6332,7 +6332,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-java"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -6344,7 +6344,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-nu"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "nu-parser",
@@ -6355,7 +6355,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-php"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -6366,7 +6366,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -6378,7 +6378,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-asset"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -6389,7 +6389,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -6411,7 +6411,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-r"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -6423,7 +6423,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ruby"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6437,7 +6437,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-rust"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "convert_case",
@@ -6454,7 +6454,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6467,7 +6467,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql-asset"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde",
@@ -6479,7 +6479,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6497,7 +6497,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts-asset"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde-wasm-bindgen",
@@ -6513,7 +6513,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wac"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -6529,7 +6529,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wasm"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "getrandom 0.2.17",
@@ -6561,7 +6561,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-yaml"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "serde",
@@ -6572,7 +6572,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-types"
-version = "1.698.0"
+version = "1.699.0"
 dependencies = [
  "anyhow",
  "bitflags",
diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml
index 43a8bd59ef..80d8c63983 100644
--- a/backend/parsers/windmill-parser-wasm/Cargo.toml
+++ b/backend/parsers/windmill-parser-wasm/Cargo.toml
@@ -12,7 +12,7 @@ resolver = "2"
 members = ["."]
 
 [workspace.package]
-version = "1.698.0"
+version = "1.699.0"
 edition = "2021"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 
diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml
index 7cb96a3e29..c6a7de2601 100644
--- a/backend/windmill-api/openapi.yaml
+++ b/backend/windmill-api/openapi.yaml
@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.698.0
+  version: 1.699.0
   title: Windmill API
 
   contact:
diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts
index a20c1b18af..3d7b371b24 100644
--- a/benchmarks/lib.ts
+++ b/benchmarks/lib.ts
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.698.0";
+export const VERSION = "v1.699.0";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({
diff --git a/cli/src/main.ts b/cli/src/main.ts
index 9650032679..372087c69a 100755
--- a/cli/src/main.ts
+++ b/cli/src/main.ts
@@ -79,7 +79,7 @@ export {
   token,
 };
 
-export const VERSION = "1.698.0";
+export const VERSION = "1.699.0";
 
 // Re-exported from constants.ts to maintain backwards compatibility
 export { WM_FORK_PREFIX } from "./core/constants.ts";
diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index c616a8e212..0ce5dc49ef 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -1,12 +1,12 @@
 {
 	"name": "windmill-components",
-	"version": "1.698.0",
+	"version": "1.699.0",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "windmill-components",
-			"version": "1.698.0",
+			"version": "1.699.0",
 			"hasInstallScript": true,
 			"license": "AGPL-3.0",
 			"dependencies": {
diff --git a/frontend/package.json b/frontend/package.json
index e6d623791f..82c7ad8fa9 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "windmill-components",
-	"version": "1.698.0",
+	"version": "1.699.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
diff --git a/lsp/Pipfile b/lsp/Pipfile
index efe961555a..3e38043b9e 100644
--- a/lsp/Pipfile
+++ b/lsp/Pipfile
@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-wmill = ">=1.698.0"
+wmill = ">=1.699.0"
 sendgrid = "*"
 mysql-connector-python = "*"
 pymongo = "*"
diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml
index d7e823897c..824243b2af 100644
--- a/openflow.openapi.yaml
+++ b/openflow.openapi.yaml
@@ -1,7 +1,7 @@
 openapi: '3.0.3'
 
 info:
-  version: 1.698.0
+  version: 1.699.0
   title: OpenFlow Spec
   contact:
     name: Ruben Fiszel
diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1
index 2e25edb52d..244a2e0d54 100644
--- a/powershell-client/WindmillClient/WindmillClient.psd1
+++ b/powershell-client/WindmillClient/WindmillClient.psd1
@@ -12,7 +12,7 @@
     RootModule        = 'WindmillClient.psm1'
 
     # Version number of this module.
-    ModuleVersion     = '1.698.0'
+    ModuleVersion     = '1.699.0'
 
     # Supported PSEditions
     # CompatiblePSEditions = @()
diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml
index d1ac989649..5aa17d46c3 100644
--- a/python-client/wmill/pyproject.toml
+++ b/python-client/wmill/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wmill"
-version = "1.698.0"
+version = "1.699.0"
 description = "A client library for accessing Windmill server wrapping the Windmill client API"
 license = "Apache-2.0"
 homepage = "https://windmill.dev"
diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json
index 07b5df0602..884ac84799 100644
--- a/typescript-client/jsr.json
+++ b/typescript-client/jsr.json
@@ -1,6 +1,6 @@
 {
   "name": "@windmill/windmill",
-  "version": "1.698.0",
+  "version": "1.699.0",
   "exports": "./src/index.ts",
   "publish": {
     "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"]
diff --git a/typescript-client/package.json b/typescript-client/package.json
index 566e4bd4e4..ec86e6d07d 100644
--- a/typescript-client/package.json
+++ b/typescript-client/package.json
@@ -1,7 +1,7 @@
 {
   "name": "windmill-client",
   "description": "Windmill SDK client for browsers and Node.js",
-  "version": "1.698.0",
+  "version": "1.699.0",
   "author": "Ruben Fiszel",
   "license": "Apache 2.0",
   "sideEffects": false,
diff --git a/version.txt b/version.txt
index 7185d214b0..dc6966fffd 100644
--- a/version.txt
+++ b/version.txt
@@ -1 +1 @@
-1.698.0
+1.699.0

From 43b18006f32fd5db54bbf8ae7ff0e0b314a517e5 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Sat, 9 May 2026 10:38:47 +0200
Subject: [PATCH 16/17] feat(cli): auto-infer args for `wmill app push` (#9091)

Run `wmill app push` from inside an app folder (e.g. `f/foo/my_app.app/`)
with no args. The local path defaults to CWD, and the remote path is
derived from CWD relative to `wmill.yaml`, with `.app`/`.raw_app`/
`__app`/`__raw_app` suffixes stripped. Either, both, or neither
positional argument can be passed.

Also resolves `file_path` against the user's original CWD before
`resolveWorkspace` may chdir to the wmill.yaml root, so a relative
`file_path` argument is interpreted from where the user invoked the
command (previously it could resolve against the wrong directory).

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 cli/src/commands/app/app.ts                   | 131 +++++++++++++++---
 cli/src/guidance/skills.gen.ts                |   2 +-
 .../auto-generated/cli/cli-commands.md        |   2 +-
 system_prompts/auto-generated/prompts.ts      |   2 +-
 .../skills/cli-commands/SKILL.md              |   2 +-
 5 files changed, 117 insertions(+), 22 deletions(-)

diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts
index f99df84636..2749c0269c 100644
--- a/cli/src/commands/app/app.ts
+++ b/cli/src/commands/app/app.ts
@@ -4,7 +4,7 @@ import { Command } from "@cliffy/command";
 import { Table } from "@cliffy/table";
 import { colors } from "@cliffy/ansi/colors";
 import * as log from "../../core/log.ts";
-import { sep as SEP } from "node:path";
+import { sep as SEP, isAbsolute, resolve as pathResolve, relative as pathRelative, basename } from "node:path";
 import { stat } from "node:fs/promises";
 import * as windmillUtils from "@windmill-labs/shared-utils";
 import { yamlParseFile } from "../../utils/yaml.ts";
@@ -12,6 +12,7 @@ import * as wmill from "../../../gen/services.gen.ts";
 import { ListableApp, Policy } from "../../../gen/types.gen.ts";
 
 import { GlobalOptions, isSuperset } from "../../types.ts";
+import { getWmillYamlPath } from "../../core/conf.ts";
 import { readInlinePathSync } from "../../utils/utils.ts";
 import devCommand from "./dev.ts";
 import lintCommand from "./lint.ts";
@@ -261,31 +262,122 @@ async function get(opts: GlobalOptions & { json?: boolean }, path: string) {
   }
 }
 
-async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
-  if (!validatePath(remotePath)) {
-    return;
-  }
+const APP_FOLDER_SUFFIXES = ["__raw_app", ".raw_app", "__app", ".app"] as const;
+
+async function push(
+  opts: GlobalOptions,
+  filePath?: string,
+  remotePath?: string
+) {
+  // Capture original CWD before resolveWorkspace, which may chdir to the
+  // wmill.yaml root. We need it to resolve relative inputs and to derive
+  // the remote path from the user's location when auto-inferring.
+  const originalCwd = process.cwd();
+
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  // Detect raw apps by checking for raw_app.yaml or __raw_app/.raw_app suffix
-  const normalizedPath = filePath.endsWith(SEP) ? filePath.slice(0, -1) : filePath;
-  const isRawApp = normalizedPath.endsWith("__raw_app") || normalizedPath.endsWith(".raw_app");
+  // Auto-infer file path from CWD when omitted
+  if (!filePath) {
+    filePath = originalCwd;
+  }
+  const absoluteFilePath = isAbsolute(filePath)
+    ? filePath
+    : pathResolve(originalCwd, filePath);
+
+  // Detect app folder type (regular vs raw)
+  const normalizedPath = absoluteFilePath.endsWith(SEP)
+    ? absoluteFilePath.slice(0, -1)
+    : absoluteFilePath;
+  const dirName = basename(normalizedPath);
+  const isRawAppByName =
+    dirName.endsWith("__raw_app") || dirName.endsWith(".raw_app");
+  const isAppByName = dirName.endsWith("__app") || dirName.endsWith(".app");
+
   let hasRawAppYaml = false;
-  if (!isRawApp) {
+  let hasAppYaml = false;
+  try {
+    await stat(normalizedPath + SEP + "raw_app.yaml");
+    hasRawAppYaml = true;
+  } catch { /* not a raw app */ }
+  if (!hasRawAppYaml) {
     try {
-      const rawAppPath = (filePath.endsWith(SEP) ? filePath : filePath + SEP) + "raw_app.yaml";
-      await stat(rawAppPath);
-      hasRawAppYaml = true;
-    } catch { /* not a raw app */ }
+      await stat(normalizedPath + SEP + "app.yaml");
+      hasAppYaml = true;
+    } catch { /* not an app */ }
   }
 
-  if (isRawApp || hasRawAppYaml) {
+  if (!isRawAppByName && !isAppByName && !hasRawAppYaml && !hasAppYaml) {
+    log.error(
+      colors.red(
+        `'${filePath}' is not an app folder (no app.yaml or raw_app.yaml, and not a *.app/*.raw_app folder).`
+      )
+    );
+    return;
+  }
+
+  // Auto-infer remote path from the folder location relative to wmill.yaml root
+  if (!remotePath) {
+    const wmillYamlPath = getWmillYamlPath();
+    if (!wmillYamlPath) {
+      log.error(
+        colors.red(
+          "Could not infer remote path: no wmill.yaml found. Run 'wmill init' or pass <remote_path> explicitly."
+        )
+      );
+      return;
+    }
+    // After resolveWorkspace, process.cwd() is the wmill.yaml dir
+    const wmillRoot = process.cwd();
+    let inferred = pathRelative(wmillRoot, normalizedPath).replaceAll(SEP, "/");
+    if (inferred.startsWith("..") || isAbsolute(inferred)) {
+      log.error(
+        colors.red(
+          `Could not infer remote path: '${filePath}' is outside the wmill.yaml root (${wmillRoot}). Move the folder under the root or pass <remote_path> explicitly.`
+        )
+      );
+      return;
+    }
+    for (const suffix of APP_FOLDER_SUFFIXES) {
+      if (inferred.endsWith(suffix)) {
+        inferred = inferred.slice(0, -suffix.length);
+        break;
+      }
+    }
+    if (!inferred) {
+      log.error(
+        colors.red(
+          "Could not infer remote path: app folder is at the wmill.yaml root. Pass <remote_path> explicitly."
+        )
+      );
+      return;
+    }
+    if (
+      !inferred.startsWith("u/") &&
+      !inferred.startsWith("g/") &&
+      !inferred.startsWith("f/")
+    ) {
+      log.error(
+        colors.red(
+          `Could not infer remote path: '${inferred}' is not under u/, g/, or f/. Move the app under one of these prefixes or pass <remote_path> explicitly.`
+        )
+      );
+      return;
+    }
+    remotePath = inferred;
+    log.info(colors.gray(`Inferred remote path: ${remotePath}`));
+  }
+
+  if (!validatePath(remotePath)) {
+    return;
+  }
+
+  if (isRawAppByName || hasRawAppYaml) {
     const { pushRawApp } = await import("./raw_apps.ts");
-    await pushRawApp(workspace.workspaceId, remotePath, filePath);
+    await pushRawApp(workspace.workspaceId, remotePath, absoluteFilePath);
     log.info(colors.bold.underline.green("Raw app pushed"));
   } else {
-    await pushApp(workspace.workspaceId, remotePath, filePath);
+    await pushApp(workspace.workspaceId, remotePath, absoluteFilePath);
     log.info(colors.bold.underline.green("App pushed"));
   }
 }
@@ -301,8 +393,11 @@ const command = new Command()
   .arguments("<path:string>")
   .option("--json", "Output as JSON (for piping to jq)")
   .action(get as any)
-  .command("push", "push a local app ")
-  .arguments("<file_path:string> <remote_path:string>")
+  .command(
+    "push",
+    "push a local app. With no args, infers the app from the current directory and the remote path from its location relative to wmill.yaml."
+  )
+  .arguments("[file_path:string] [remote_path:string]")
   .action(push as any)
   .command("dev", devCommand)
   .command("lint", lintCommand)
diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts
index ede19be85b..72d2e9d76b 100644
--- a/cli/src/guidance/skills.gen.ts
+++ b/cli/src/guidance/skills.gen.ts
@@ -6588,7 +6588,7 @@ app related commands
   - \`--json\` - Output as JSON (for piping to jq)
 - \`app get <path:string>\` - get an app's details
   - \`--json\` - Output as JSON (for piping to jq)
-- \`app push <file_path:string> <remote_path:string>\` - push a local app 
+- \`app push [file_path:string] [remote_path:string]\` - push a local app. With no args, infers the app from the current directory and the remote path from its location relative to wmill.yaml.
 - \`app dev [app_folder:string]\` - Start a development server for building apps with live reload and hot module replacement
   - \`--port <port:number>\` - Port to run the dev server on (will find next available port if occupied)
   - \`--host <host:string>\` - Host to bind the dev server to
diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md
index 692e2dd310..dc0eb66640 100644
--- a/system_prompts/auto-generated/cli/cli-commands.md
+++ b/system_prompts/auto-generated/cli/cli-commands.md
@@ -26,7 +26,7 @@ app related commands
   - `--json` - Output as JSON (for piping to jq)
 - `app get <path:string>` - get an app's details
   - `--json` - Output as JSON (for piping to jq)
-- `app push <file_path:string> <remote_path:string>` - push a local app 
+- `app push [file_path:string] [remote_path:string]` - push a local app. With no args, infers the app from the current directory and the remote path from its location relative to wmill.yaml.
 - `app dev [app_folder:string]` - Start a development server for building apps with live reload and hot module replacement
   - `--port <port:number>` - Port to run the dev server on (will find next available port if occupied)
   - `--host <host:string>` - Host to bind the dev server to
diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts
index 83f6ae1157..53336762cc 100644
--- a/system_prompts/auto-generated/prompts.ts
+++ b/system_prompts/auto-generated/prompts.ts
@@ -2174,7 +2174,7 @@ app related commands
   - \`--json\` - Output as JSON (for piping to jq)
 - \`app get <path:string>\` - get an app's details
   - \`--json\` - Output as JSON (for piping to jq)
-- \`app push <file_path:string> <remote_path:string>\` - push a local app 
+- \`app push [file_path:string] [remote_path:string]\` - push a local app. With no args, infers the app from the current directory and the remote path from its location relative to wmill.yaml.
 - \`app dev [app_folder:string]\` - Start a development server for building apps with live reload and hot module replacement
   - \`--port <port:number>\` - Port to run the dev server on (will find next available port if occupied)
   - \`--host <host:string>\` - Host to bind the dev server to
diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md
index b4818b4fa1..98ce52efde 100644
--- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md
+++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md
@@ -31,7 +31,7 @@ app related commands
   - `--json` - Output as JSON (for piping to jq)
 - `app get <path:string>` - get an app's details
   - `--json` - Output as JSON (for piping to jq)
-- `app push <file_path:string> <remote_path:string>` - push a local app 
+- `app push [file_path:string] [remote_path:string]` - push a local app. With no args, infers the app from the current directory and the remote path from its location relative to wmill.yaml.
 - `app dev [app_folder:string]` - Start a development server for building apps with live reload and hot module replacement
   - `--port <port:number>` - Port to run the dev server on (will find next available port if occupied)
   - `--host <host:string>` - Host to bind the dev server to

From b95f0e2379a82b7064d9317dfacac852b1988699 Mon Sep 17 00:00:00 2001
From: Ruben Fiszel <ruben@windmill.dev>
Date: Sun, 10 May 2026 12:06:25 +0000
Subject: [PATCH 17/17] style(log-viewer): tighter top bar (icons, hyphenated
 label, scoped overflow) (#9094)

- duration/mem-peak labels replaced by Timer/Cpu icons with tooltips
- 'Auto scroll' -> 'auto-scroll' (lowercase, hyphenated, whitespace-nowrap)
- top bar gets overflow-x-auto so it scrolls horizontally instead of pushing the entire log panel into overflow when narrow
---
 frontend/src/lib/components/LogViewer.svelte | 36 ++++++++++++++------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/frontend/src/lib/components/LogViewer.svelte b/frontend/src/lib/components/LogViewer.svelte
index 9f06f90eec..e3012a9db6 100644
--- a/frontend/src/lib/components/LogViewer.svelte
+++ b/frontend/src/lib/components/LogViewer.svelte
@@ -11,7 +11,7 @@
 </script>
 
 <script lang="ts">
-	import { ClipboardCopy, Download, Expand, Loader2 } from 'lucide-svelte'
+	import { ClipboardCopy, Download, Expand, Loader2, Timer, Cpu } from 'lucide-svelte'
 	import { Button, Drawer, DrawerContent } from './common'
 	import { copyToClipboard } from '$lib/utils'
 	import { base } from '$lib/base'
@@ -166,8 +166,7 @@
 	}
 
 	export function scrollToBottom() {
-		scroll &&
-			setTimeout(() => preEl?.scroll({ top: preEl?.scrollHeight, behavior: 'smooth' }), 100)
+		scroll && setTimeout(() => preEl?.scroll({ top: preEl?.scrollHeight, behavior: 'smooth' }), 100)
 	}
 
 	let logViewer: Drawer | undefined = $state()
@@ -296,7 +295,9 @@
 			class="w-full h-full bg-surface-secondary flex flex-col {noMaxH ? '' : 'max-h-screen'}"
 			data-nav-id={navigationId}
 		>
-			<div class="flex gap-2 ml-2 {small ? 'py-1' : 'py-2'} border-b">
+			<div
+				class="flex gap-2 ml-2 {small ? 'py-1' : 'py-2'} border-b overflow-x-auto overflow-y-hidden"
+			>
 				{#if isLoading}
 					<div class="flex gap-2 items-center">
 						<Loader2 class="animate-spin" />
@@ -312,14 +313,27 @@
 					</div>
 				{:else if duration}
 					<span
-						class={twMerge('text-secondary dark:text-gray-400', small ? '!text-2xs' : '!text-xs')}
-						>took {duration}ms</span
+						class={twMerge(
+							'flex items-center gap-1 text-secondary dark:text-gray-400',
+							small ? '!text-2xs' : '!text-xs'
+						)}
+						title="Duration"
 					>
+						<Timer size={small ? 10 : 12} />
+						{duration}ms
+					</span>
 				{/if}
 				{#if mem}
-					<span class="{small ? '!text-2xs' : '!text-xs'} text-secondary dark:text-gray-400"
-						>mem peak: {(mem / 1024).toPrecision(4)}MB</span
+					<span
+						class={twMerge(
+							'flex items-center gap-1 text-secondary dark:text-gray-400',
+							small ? '!text-2xs' : '!text-xs'
+						)}
+						title="Memory peak"
 					>
+						<Cpu size={small ? 10 : 12} />
+						{(mem / 1024).toPrecision(4)}MB
+					</span>
 				{/if}
 				<div class="flex gap-2 justify-end flex-1">
 					{#if jobId && download}
@@ -335,8 +349,10 @@
 					{/if}
 					<button onclick={logViewer.openDrawer}><Expand size="12" /></button>
 					{#if !noAutoScroll}
-						<label class="pr-2 text-2xs flex gap-2 font-normal text-primary items-center">
-							Auto scroll
+						<label
+							class="pr-2 text-2xs flex gap-2 font-normal text-primary items-center whitespace-nowrap"
+						>
+							auto-scroll
 							<input class="windmillapp" type="checkbox" bind:checked={scroll} />
 						</label>
 					{/if}