From 532cdc4b49e2df0920ff6aed35c0da41ba78aa82 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 16 Jul 2025 15:26:45 +0200 Subject: [PATCH 01/22] chore(main): release 1.507.2 (#6202) * chore(main): release 1.507.2 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 8 +++ backend/Cargo.lock | 54 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 52 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29f5b49b55..6769e30eb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.507.2](https://github.com/windmill-labs/windmill/compare/v1.507.1...v1.507.2) (2025-07-16) + + +### Bug Fixes + +* cancel autocomplete on escape + autocomplete qol ([#6201](https://github.com/windmill-labs/windmill/issues/6201)) ([038c179](https://github.com/windmill-labs/windmill/commit/038c179a432d4f1997a761392e27b16dea829e06)) +* **cli:** fix cli --skip-resources --skip-variables ([6c477c1](https://github.com/windmill-labs/windmill/commit/6c477c109bf7e803ec07d4485dec426b770c7f76)) + ## [1.507.1](https://github.com/windmill-labs/windmill/compare/v1.507.0...v1.507.1) (2025-07-16) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e95a7b9d82..26a815cc71 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14676,7 +14676,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "axum", @@ -14728,7 +14728,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "argon2", @@ -14839,7 +14839,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.507.1" +version = "1.507.2" dependencies = [ "base64 0.22.1", "chrono", @@ -14854,7 +14854,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.507.1" +version = "1.507.2" dependencies = [ "chrono", "serde", @@ -14867,7 +14867,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "serde", @@ -14881,7 +14881,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "async-stream", @@ -14959,7 +14959,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.507.1" +version = "1.507.2" dependencies = [ "regex", "serde", @@ -14973,7 +14973,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "bytes", @@ -14996,7 +14996,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.507.1" +version = "1.507.2" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15008,7 +15008,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.507.1" +version = "1.507.2" dependencies = [ "convert_case 0.6.0", "serde", @@ -15017,7 +15017,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "lazy_static", @@ -15029,7 +15029,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "serde_json", @@ -15041,7 +15041,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "gosyn", @@ -15053,7 +15053,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "lazy_static", @@ -15065,7 +15065,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "serde_json", @@ -15077,7 +15077,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "nu-parser", @@ -15088,7 +15088,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15099,7 +15099,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15111,7 +15111,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "async-recursion", @@ -15134,7 +15134,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15151,7 +15151,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "lazy_static", @@ -15165,7 +15165,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "lazy_static", @@ -15183,7 +15183,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -15207,7 +15207,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "serde_json", @@ -15217,7 +15217,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "async-recursion", @@ -15250,7 +15250,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.507.1" +version = "1.507.2" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15260,7 +15260,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.507.1" +version = "1.507.2" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a0a79b8385..ed5d1c7ec5 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.507.1" +version = "1.507.2" authors.workspace = true edition.workspace = true @@ -32,7 +32,7 @@ members = [ ] [workspace.package] -version = "1.507.1" +version = "1.507.2" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 9b1e34e954..17f54be854 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.507.1 + version: 1.507.2 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index bf488b171f..7578655f4c 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.507.1"; +export const VERSION = "v1.507.2"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index aae8ea4653..b298d2a7ad 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -65,7 +65,7 @@ export { // } // }); -export const VERSION = "1.507.1"; +export const VERSION = "1.507.2"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4ff437e93f..d93996431f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.507.1", + "version": "1.507.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.507.1", + "version": "1.507.2", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 012336c1f3..8f821a6c7e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.507.1", + "version": "1.507.2", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index fc372bb3a5..1a98c5d097 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.507.1" -wmill_pg = ">=1.507.1" +wmill = ">=1.507.2" +wmill_pg = ">=1.507.2" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 3eb17a5326..cf6b5fa467 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.507.1 + version: 1.507.2 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index c5033eb071..fb7f0db353 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.507.1' + ModuleVersion = '1.507.2' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index a5760b2f56..61d25dd1a4 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.507.1" +version = "1.507.2" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 987e5ec3a9..4e0e26fe74 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.507.1" +version = "1.507.2" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 6f071b168b..ab61fd2b7f 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.507.1", + "version": "1.507.2", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 1417a4f461..e19ab5e2a9 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.507.1", + "version": "1.507.2", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 0f5f07a749..5f5962f115 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.507.1 +1.507.2 From 9e8748b7c82be0a719680bf36f4c906002e0dd38 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Wed, 16 Jul 2025 16:30:15 +0200 Subject: [PATCH 02/22] fix batch rerun reactivity issue (#6205) --- .../src/lib/components/runs/BatchReRunOptionsPane.svelte | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/runs/BatchReRunOptionsPane.svelte b/frontend/src/lib/components/runs/BatchReRunOptionsPane.svelte index 1c3eed8e60..6095f086c1 100644 --- a/frontend/src/lib/components/runs/BatchReRunOptionsPane.svelte +++ b/frontend/src/lib/components/runs/BatchReRunOptionsPane.svelte @@ -28,6 +28,7 @@ } from '$lib/components/jobs/batchReruns' import Toggle from '../Toggle.svelte' import { TriangleAlert } from 'lucide-svelte' + import { readFieldsRecursively } from '$lib/utils' let { selectedIds, @@ -149,7 +150,10 @@ (options[selected.kind][selected.script_path]?.use_latest_version ?? false)) ) - const jobGroupsPromise = $derived(selectedIds && untrack(() => fetchJobGroups())) + const jobGroupsPromise = $derived.by(() => { + readFieldsRecursively(selectedIds) + return untrack(() => fetchJobGroups()) + })
From 290daec0fa53f77f6ad02e746d82aa30997d8cb0 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 16 Jul 2025 18:44:06 +0200 Subject: [PATCH 03/22] fix(frontend): enable delete triggers on detail page (#6206) * Allways show trigger update button * allow delete deployed trigger --- .../triggers/DeleteTriggerButton.svelte | 2 +- .../triggers/TriggerEditorToolbar.svelte | 6 +-- .../components/triggers/TriggersEditor.svelte | 52 +++++++++++++++++-- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/frontend/src/lib/components/triggers/DeleteTriggerButton.svelte b/frontend/src/lib/components/triggers/DeleteTriggerButton.svelte index d65c26a01d..b06392bb7e 100644 --- a/frontend/src/lib/components/triggers/DeleteTriggerButton.svelte +++ b/frontend/src/lib/components/triggers/DeleteTriggerButton.svelte @@ -18,7 +18,7 @@ { diff --git a/frontend/src/lib/components/triggers/TriggerEditorToolbar.svelte b/frontend/src/lib/components/triggers/TriggerEditorToolbar.svelte index b54c80e931..ac3032afc1 100644 --- a/frontend/src/lib/components/triggers/TriggerEditorToolbar.svelte +++ b/frontend/src/lib/components/triggers/TriggerEditorToolbar.svelte @@ -85,7 +85,7 @@ />
{/if} - {#if trigger?.isDraft} + {#if trigger?.isDraft || permissions === 'create'} {:else if !trigger?.isDraft && trigger?.draftConfig} + {#if $userStore} + + {/if} {:else if !result?.disable_download} - + {#if $userStore} + + {/if} {/if} {#if typeof result?.s3 === 'string'} diff --git a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte index a12fb6b421..c54ab21838 100644 --- a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte +++ b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte @@ -58,11 +58,7 @@ allowMultiple = true, allowDelete = true, folderOnly = false, - containerText = folderOnly - ? 'Drag and drop a folder here or click to browse' - : allowMultiple - ? 'Drag and drop files here or click to browse' - : 'Drag and drop a file here or click to browse', + containerText: customContainerText = undefined, customResourcePath = undefined, customResourceType = undefined, customClass = '', @@ -80,6 +76,15 @@ computeForceViewerPolicies = undefined }: Props = $props() + const containerText = $derived( + customContainerText ?? + (folderOnly + ? 'Drag and drop a folder here or click to browse' + : allowMultiple + ? 'Drag and drop files here or click to browse' + : 'Drag and drop a file here or click to browse') + ) + const dispatch = createEventDispatcher<{ addition: { path?: string; filename?: string } deletion: { path: string } diff --git a/frontend/src/lib/components/common/fileUpload/S3ArgInput.svelte b/frontend/src/lib/components/common/fileUpload/S3ArgInput.svelte index 0f072e99ef..f87fc64851 100644 --- a/frontend/src/lib/components/common/fileUpload/S3ArgInput.svelte +++ b/frontend/src/lib/components/common/fileUpload/S3ArgInput.svelte @@ -5,6 +5,7 @@ import FileUpload from './FileUpload.svelte' import type SimpleEditor from '$lib/components/SimpleEditor.svelte' import Button from '../button/Button.svelte' + import { userStore } from '$lib/stores' let { multiple, @@ -45,24 +46,26 @@ let fileUpload: FileUpload | undefined = $state(undefined) - { - if (multiple) { - if (Array.isArray(value)) { - value.push(ev.detail) +{#if $userStore} + { + if (multiple) { + if (Array.isArray(value)) { + value.push(ev.detail) + } else { + value = [ev.detail] + } + fileUpload?.addUpload(ev.detail) } else { - value = [ev.detail] + value = ev.detail + fileUpload?.setUpload(value) } - fileUpload?.addUpload(ev.detail) - } else { - value = ev.detail - fileUpload?.setUpload(value) - } - editor?.setCode(JSON.stringify(value)) - }} - readOnlyMode={false} -/> + editor?.setCode(JSON.stringify(value)) + }} + readOnlyMode={false} + /> +{/if}
{/if} - + {#if $userStore} + + {/if}
From 1d62dd0fc323e484ab1d08e740379c8243b52df8 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Wed, 16 Jul 2025 19:26:37 +0200 Subject: [PATCH 05/22] feat: better explanation for dev key renewal (#6209) * fix: hide s3 catalog picker in anonymous apps * nit reactivity upload text * no signed s3 object warning * remove warning * fix check * feat: better dev key explanation --- .../src/lib/components/InstanceSetting.svelte | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 51cbe3452f..f724e97a1a 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -406,9 +406,16 @@ {:else if expiration}
- License key expired on {expiration} + + {#if $values[setting.key]?.endsWith('__dev')} + Dev license key expired on {expiration}.
If even after successful + renewal, your dev license key is still expired, it means that your production + key has expired due to unpaid invoices or excessive use of your production + instance. + {:else} + License key expired on {expiration}. + {/if} +
{:else}
@@ -440,12 +447,17 @@ : 'text-red-600' )} > - {latestKeyRenewalAttempt.result === 'success' - ? 'Latest key renewal succeeded' - : isTrial - ? 'Latest key renewal ignored because in trial' - : 'Latest key renewal failed'} - on {attemptedAt} + {#if latestKeyRenewalAttempt.result === 'success' && $values[setting.key]?.endsWith('__dev')} + Latest dev key renewal succeeded on {attemptedAt}. The dev key expiry was + updated to align with your current production key's expiration date. + {:else} + {latestKeyRenewalAttempt.result === 'success' + ? 'Latest key renewal succeeded' + : isTrial + ? 'Latest key renewal ignored because in trial' + : 'Latest key renewal failed'} + on {attemptedAt} + {/if}
{#snippet text()} From c076d1332e0871b16b5881e2cf3ef165bc9f5dd5 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 16 Jul 2025 19:26:48 +0200 Subject: [PATCH 06/22] fix(frontend): add error handler on trigger delete (#6208) --- .../components/triggers/TriggersEditor.svelte | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/frontend/src/lib/components/triggers/TriggersEditor.svelte b/frontend/src/lib/components/triggers/TriggersEditor.svelte index e0b0109d7d..c054bf9adb 100644 --- a/frontend/src/lib/components/triggers/TriggersEditor.svelte +++ b/frontend/src/lib/components/triggers/TriggersEditor.svelte @@ -31,6 +31,7 @@ GcpTriggerService, SqsTriggerService } from '$lib/gen' + import { sendUserToast } from '$lib/toast' interface Props { noEditor: boolean @@ -88,6 +89,7 @@ triggersState.selectedTriggerIndex = triggerIndex } + let deletingTrigger = $state(undefined) async function deleteDeployedTrigger(triggerIndex: number) { const { type: triggerType, path: triggerPath } = triggersState.triggers[triggerIndex] if (!triggerPath) { @@ -107,11 +109,19 @@ } const deleteHandler = deleteHandlers[triggerType as keyof typeof deleteHandlers] - if (deleteHandler) { - await deleteHandler()({ - workspace: $workspaceStore ?? '', - path: triggerPath ?? '' - }) + if (deleteHandler && deletingTrigger !== triggerIndex) { + deletingTrigger = triggerIndex + try { + await deleteHandler()({ + workspace: $workspaceStore ?? '', + path: triggerPath ?? '' + }) + sendUserToast(`Successfully deleted ${triggerType} trigger: ${triggerPath}`) + } catch (error) { + sendUserToast(error.body || error.message, true) + } finally { + deletingTrigger = undefined + } } } From 827e06b4b3dd7b2b970b3a0a37e3dfbe98d1fafd Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 16 Jul 2025 19:59:47 +0200 Subject: [PATCH 07/22] feat(aichat): add logs api endpoint as tool (#6197) * add logs * add logs search + better load tools logic * use json * nit * only add for ee * nit * filter out search after first fail * Revert "filter out search after first fail" This reverts commit 2abf0db6e5a1be84e67d1a153281b74d448cb5cd. * call endpoint to know if it is available * cleaning * Apply suggestion from @graphite-app[bot] Co-authored-by: graphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com> * fix * draft * call enabled endpoint * not workspaced * remove from system prompt if not enterprise * fix eeref command * update ee ref --------- Co-authored-by: graphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com> --- .github/workflows/git-commands.yaml | 10 +- backend/ee-repo-ref.txt | 2 +- .../copilot/chat/AIChatManager.svelte.ts | 23 +-- .../copilot/chat/navigator/apiTools.ts | 142 ++++++++++++++++-- .../components/copilot/chat/navigator/core.ts | 3 +- 5 files changed, 149 insertions(+), 31 deletions(-) diff --git a/.github/workflows/git-commands.yaml b/.github/workflows/git-commands.yaml index 4dbdf36ed1..9dcae33bb1 100644 --- a/.github/workflows/git-commands.yaml +++ b/.github/workflows/git-commands.yaml @@ -151,17 +151,17 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - echo "${{ steps.get-commit-hash.outputs.commit_hash }}" > backend/ee-repo-ref.txt - echo "Updated backend/ee-repo-ref.txt with commit hash: ${{ steps.get-commit-hash.outputs.commit_hash }}" - # commit and push the changes - PR_NUMBER=${{ github.event.issue.number }} - BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) echo "Checking out PR branch: $BRANCH_NAME" git checkout $BRANCH_NAME git config --local user.email "action@github.com" git config --local user.name "GitHub Action" git config pull.rebase true git pull origin $BRANCH_NAME + echo "${{ steps.get-commit-hash.outputs.commit_hash }}" > backend/ee-repo-ref.txt + echo "Updated backend/ee-repo-ref.txt with commit hash: ${{ steps.get-commit-hash.outputs.commit_hash }}" + # commit and push the changes + PR_NUMBER=${{ github.event.issue.number }} + BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) git add backend/ee-repo-ref.txt git commit -m "Update ee-repo-ref.txt" || echo "No changes to commit" git push origin $BRANCH_NAME diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 653071700b..3c15148a60 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -3cccbe81494f7edd113dce2812278b00c91160c0 \ No newline at end of file +64f3cdcd831b81ca42991ddc201533753d590a6b diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 2983d4b35b..78282eca09 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -89,15 +89,16 @@ class AIChatManager { open = $derived(chatState.size > 0) - constructor() { - loadApiTools() - .then((tools) => { - this.apiTools = tools - }) - .catch((err) => { - console.error('Error loading api tools', err) - this.apiTools = [] - }) + loadApiTools = async () => { + try { + this.apiTools = await loadApiTools() + if (this.mode === AIMode.NAVIGATOR) { + this.tools = [this.changeModeTool, ...navigatorTools, ...this.apiTools] + } + } catch (err) { + console.error('Error loading api tools', err) + this.apiTools = [] + } } setAiChatInput(aiChatInput: AIChatInput | null) { @@ -604,6 +605,10 @@ class AIChatManager { } } + if (this.mode === AIMode.NAVIGATOR && this.apiTools.length === 0) { + await this.loadApiTools() + } + await this.chatRequest({ ...params }) diff --git a/frontend/src/lib/components/copilot/chat/navigator/apiTools.ts b/frontend/src/lib/components/copilot/chat/navigator/apiTools.ts index f03920415f..1aea113ba6 100644 --- a/frontend/src/lib/components/copilot/chat/navigator/apiTools.ts +++ b/frontend/src/lib/components/copilot/chat/navigator/apiTools.ts @@ -1,7 +1,7 @@ import type { ChatCompletionTool } from 'openai/resources/index.mjs' import type { Tool } from '../shared' import { get } from 'svelte/store' -import { workspaceStore } from '$lib/stores' +import { workspaceStore, enterpriseLicense } from '$lib/stores' // OpenAPI type definitions interface OpenAPIParameter { @@ -54,6 +54,88 @@ interface OpenAPISpec { paths: { [path: string]: OpenAPIPathItem } + components?: { + parameters?: { + [name: string]: OpenAPIParameter + } + schemas?: { + [name: string]: any + } + } +} + +interface OpenAPIParameterWithRef { + $ref?: string + name?: string + in?: string + description?: string + required?: boolean + schema?: { + type?: string + format?: string + } +} + +/** + * Dereferences parameter $ref references in an OpenAPI spec + * Only resolves parameter references, not schemas or other components + */ +function dereferenceParameters(spec: OpenAPISpec): OpenAPISpec { + if (!spec.components?.parameters) { + return spec + } + + const resolveParameterRef = (paramRef: OpenAPIParameterWithRef): OpenAPIParameter => { + if (paramRef.$ref) { + // Extract parameter name from $ref (e.g., "#/components/parameters/WorkspaceId" -> "WorkspaceId") + const refPath = paramRef.$ref.split('/') + if (refPath.length >= 4 && refPath[1] === 'components' && refPath[2] === 'parameters') { + const paramName = refPath[3] + const resolvedParam = spec.components?.parameters?.[paramName] + if (resolvedParam) { + return resolvedParam + } + } + console.warn(`Could not resolve parameter reference: ${paramRef.$ref}`) + return paramRef as OpenAPIParameter + } + return paramRef as OpenAPIParameter + } + + const processParameters = (parameters: OpenAPIParameterWithRef[]): OpenAPIParameter[] => { + return parameters.map(resolveParameterRef) + } + + const dereferencedSpec: OpenAPISpec = { + ...spec, + paths: {} + } + + // Process each path + for (const [pathKey, pathItem] of Object.entries(spec.paths)) { + const newPathItem: OpenAPIPathItem = { ...pathItem } + + // Dereference path-level parameters + if (pathItem.parameters) { + newPathItem.parameters = processParameters(pathItem.parameters as OpenAPIParameterWithRef[]) + } + + // Dereference operation-level parameters + const methods = ['get', 'post', 'put', 'delete', 'patch', 'options'] as const + for (const method of methods) { + const operation = pathItem[method] + if (operation?.parameters) { + newPathItem[method] = { + ...operation, + parameters: processParameters(operation.parameters as OpenAPIParameterWithRef[]) + } + } + } + + dereferencedSpec.paths[pathKey] = newPathItem + } + + return dereferencedSpec } const buildApiCallTools = ( @@ -124,6 +206,10 @@ export function buildToolsFromOpenApi( // Add path parameters for (const param of pathParams) { + if (param.name === 'workspace') { + continue + } + parameters.properties[param.name] = { type: param.schema?.type || 'string', description: param.description || `Path parameter: ${param.name}` @@ -167,13 +253,13 @@ export function buildToolsFromOpenApi( } const tool = buildApiCallTools( - 'api_' + op.operationId, + 'api_' + op.operationId.replace(/\s+/g, ''), op.summary || op.description || `${method.toUpperCase()} ${path}`, parameters ) // Store the endpoint path in the map - endpointMap['api_' + op.operationId] = `${method.toUpperCase()} ${path}` + endpointMap['api_' + op.operationId.replace(/\s+/g, '')] = `${method.toUpperCase()} ${path}` tools.push(tool) } @@ -193,7 +279,6 @@ export function createApiTools( const toolName = chatTool.function.name let endpoint = endpointMap[toolName] || '' endpoint = endpoint.replace('{workspace}', get(workspaceStore) as string) - toolCallbacks.setToolStatus(toolId, `Calling API endpoint (${endpoint})...`) try { // Extract method and path from endpoint @@ -212,8 +297,8 @@ export function createApiTools( if (key === 'body') continue // Body is handled separately // Check if this is a path parameter - if (url.includes(`:${key}`)) { - url = url.replace(`:${key}`, encodeURIComponent(String(value))) + if (url.includes(`{${key}}`)) { + url = url.replace(`{${key}}`, encodeURIComponent(String(value))) } else { // Assume it's a query parameter queryParams[key] = String(value) @@ -231,18 +316,32 @@ export function createApiTools( // Log the constructed URL console.log(`Calling API: ${method} ${url} with args: ${JSON.stringify(args)}`) + toolCallbacks.setToolStatus(toolId, `Calling API endpoint (${url})...`) + const response = await fetch(url, { method: method }) - const data = await response.json() - // For now, return a placeholder response - // In a real implementation, we would make the actual API call here - toolCallbacks.setToolStatus(toolId, `API call to ${endpoint} completed`) - return JSON.stringify({ - success: true, - data: data - }) + if (response.ok) { + let result = '' + if (response.headers.get('content-type')?.includes('application/json')) { + result = await response.json() + } else { + result = await response.text() + } + toolCallbacks.setToolStatus(toolId, `API call to ${url} completed`) + return JSON.stringify({ + success: true, + data: result + }) + } else { + const text = await response.text() + toolCallbacks.setToolStatus(toolId, `API call to ${url} failed`) + return JSON.stringify({ + success: false, + data: text + }) + } } catch (error) { toolCallbacks.setToolStatus(toolId, `API call to ${endpoint} failed`) console.error(`Error calling API to ${endpoint}:`, error) @@ -256,9 +355,14 @@ export function createApiTools( export async function loadApiTools(): Promise[]> { try { const response = await fetch('/api/openapi.json') - const openApiSpec = (await response.json()) as OpenAPISpec + const rawOpenApiSpec = (await response.json()) as OpenAPISpec + + // Dereference parameter references + const openApiSpec = dereferenceParameters(rawOpenApiSpec) + const pathsToInclude = [ 'jobs', + 'jobs_u', 'scripts', 'flows', 'resources', @@ -267,6 +371,14 @@ export async function loadApiTools(): Promise[]> { 'workers' ] + // call srch endpoint to check if it's available + if (get(enterpriseLicense)) { + const srchResponse = await fetch(`/api/srch/index/search/enabled`) + if (srchResponse.ok) { + pathsToInclude.push('srch/w') // job search + } + } + const { tools: apiTools, endpointMap } = buildToolsFromOpenApi(openApiSpec, { pathFilter: (path) => pathsToInclude.some((p) => path.includes(`/${p}/`)), methodFilter: ['get'] diff --git a/frontend/src/lib/components/copilot/chat/navigator/core.ts b/frontend/src/lib/components/copilot/chat/navigator/core.ts index aecf60bc30..9194ec290d 100644 --- a/frontend/src/lib/components/copilot/chat/navigator/core.ts +++ b/frontend/src/lib/components/copilot/chat/navigator/core.ts @@ -5,7 +5,7 @@ import type { } from 'openai/resources/index.mjs' import type { Tool } from '../shared' import { ResourceService } from '$lib/gen' -import { workspaceStore } from '$lib/stores' +import { enterpriseLicense, workspaceStore } from '$lib/stores' import { get } from 'svelte/store' import { triggerablesByAi } from '../sharedChatState.svelte' @@ -29,6 +29,7 @@ INSTRUCTIONS: - If you are asked to fill a form or act on an input, input the existing json object and change the fields the user asked you to change. Take into account the prompt_for_ai field of the schema to know what and how to do changes. Then tell the user that you have updated the form, and ask him to review the changes before running the script or flow. - For form inputs where format starts with "resource-" and is not "resource-obj", fetch the available resources using get_available_resources, and then use the resource_path prefixed with "$res:" to fill the input. - If you are not sure about an input, set the ones you are sure about, and then ask the user for the value of the input you are not sure about. +${get(enterpriseLicense) ? `- If asked to look through the jobs logs, use the /srch/w/{workspace}/index/search/job endpoint to search for the relevant jobs runs. Then use /w/{workspace}/jobs_u/get to get the logs of each job.` : ''} GENERAL PRINCIPLES: - Be concise but thorough From 07b785dcc383ae87eaec65e765177e45fff0af20 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 16 Jul 2025 22:31:15 +0000 Subject: [PATCH 08/22] fix: support enum for array list as multiselect --- frontend/src/lib/components/ArgInput.svelte | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index b7c274a0b4..4dc1184eca 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -648,6 +648,16 @@ reorderable /> + {:else if enum_ && (Array.isArray(value) || value == undefined)} +
+ dispatch('focus')} + reorderable + /> +
{:else if itemsType?.enum != undefined && Array.isArray(itemsType?.enum) && (Array.isArray(value) || value == undefined)}
Date: Wed, 16 Jul 2025 22:52:29 +0000 Subject: [PATCH 09/22] fix build --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 3c15148a60..4598505971 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -64f3cdcd831b81ca42991ddc201533753d590a6b +24faaae726388d275529ffb939f57d60c07196c4 \ No newline at end of file From 7ca652dede28162634752c44cf6ff2e94be19f82 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 17 Jul 2025 01:05:38 +0200 Subject: [PATCH 10/22] chore(main): release 1.508.0 (#6207) * chore(main): release 1.508.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 16 ++++ backend/Cargo.lock | 74 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 70 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6769e30eb6..9f3b540dce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [1.508.0](https://github.com/windmill-labs/windmill/compare/v1.507.2...v1.508.0) (2025-07-16) + + +### Features + +* **aichat:** add logs api endpoint as tool ([#6197](https://github.com/windmill-labs/windmill/issues/6197)) ([827e06b](https://github.com/windmill-labs/windmill/commit/827e06b4b3dd7b2b970b3a0a37e3dfbe98d1fafd)) +* better explanation for dev key renewal ([#6209](https://github.com/windmill-labs/windmill/issues/6209)) ([1d62dd0](https://github.com/windmill-labs/windmill/commit/1d62dd0fc323e484ab1d08e740379c8243b52df8)) + + +### Bug Fixes + +* **frontend:** add error handler on trigger delete ([#6208](https://github.com/windmill-labs/windmill/issues/6208)) ([c076d13](https://github.com/windmill-labs/windmill/commit/c076d1332e0871b16b5881e2cf3ef165bc9f5dd5)) +* **frontend:** enable delete triggers on detail page ([#6206](https://github.com/windmill-labs/windmill/issues/6206)) ([290daec](https://github.com/windmill-labs/windmill/commit/290daec0fa53f77f6ad02e746d82aa30997d8cb0)) +* hide s3 catalog picker in anonymous apps ([#6204](https://github.com/windmill-labs/windmill/issues/6204)) ([b68193f](https://github.com/windmill-labs/windmill/commit/b68193f80468aeeadff5af0826cca2f0cc9f05e8)) +* support enum for array list as multiselect ([07b785d](https://github.com/windmill-labs/windmill/commit/07b785dcc383ae87eaec65e765177e45fff0af20)) + ## [1.507.2](https://github.com/windmill-labs/windmill/compare/v1.507.1...v1.507.2) (2025-07-16) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 26a815cc71..8c81028a53 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -776,9 +776,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687bc16bc431a8533fe0097c7f0182874767f920989d7260950172ae8e3c4465" +checksum = "b68c2194a190e1efc999612792e25b1ab3abfefe4306494efaaabc25933c0cbe" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -835,9 +835,9 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "1.75.0" +version = "1.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c27b0a5c8f9f8db35929bde0612d738d8c872ab887232579c61d73dba7a5d17" +checksum = "837b4b181a1306a5a67d85218fa7d7298e340041d9bd81ae6919b9ec4f86ca97" dependencies = [ "aws-credential-types", "aws-runtime", @@ -857,9 +857,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.75.0" +version = "1.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36d2a4d2753a2304cc0a86cff15581179c420c7b3054c0cbc884a2e497e79ae5" +checksum = "64bf26698dd6d238ef1486bdda46f22a589dc813368ba868dc3d94c8d27b56ba" dependencies = [ "aws-credential-types", "aws-runtime", @@ -879,9 +879,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.76.0" +version = "1.77.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719f007d6914388e2f14943c663447cfc44bb9d0478bb96df891258ba5145f82" +checksum = "09cd07ed1edd939fae854a22054299ae3576500f4e0fadc560ca44f9c6ea1664" dependencies = [ "aws-credential-types", "aws-runtime", @@ -901,9 +901,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.77.0" +version = "1.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efcc3dbb630f3e0d16a4f79fcd4b3fb1c59cb89f1031c46e44976f640b5de27" +checksum = "37f7766d2344f56d10d12f3c32993da36d78217f32594fe4fb8e57a538c1cdea" dependencies = [ "aws-credential-types", "aws-runtime", @@ -14676,7 +14676,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "axum", @@ -14728,7 +14728,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "argon2", @@ -14839,7 +14839,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.507.2" +version = "1.508.0" dependencies = [ "base64 0.22.1", "chrono", @@ -14854,7 +14854,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.507.2" +version = "1.508.0" dependencies = [ "chrono", "serde", @@ -14867,7 +14867,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "serde", @@ -14881,7 +14881,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "async-stream", @@ -14959,7 +14959,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.507.2" +version = "1.508.0" dependencies = [ "regex", "serde", @@ -14973,7 +14973,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "bytes", @@ -14996,7 +14996,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.507.2" +version = "1.508.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15008,7 +15008,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.507.2" +version = "1.508.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15017,7 +15017,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "lazy_static", @@ -15029,7 +15029,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "serde_json", @@ -15041,7 +15041,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "gosyn", @@ -15053,7 +15053,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "lazy_static", @@ -15065,7 +15065,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "serde_json", @@ -15077,7 +15077,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "nu-parser", @@ -15088,7 +15088,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15099,7 +15099,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15111,7 +15111,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "async-recursion", @@ -15134,7 +15134,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15151,7 +15151,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "lazy_static", @@ -15165,7 +15165,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "lazy_static", @@ -15183,7 +15183,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -15207,7 +15207,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "serde_json", @@ -15217,7 +15217,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "async-recursion", @@ -15250,7 +15250,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.507.2" +version = "1.508.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15260,7 +15260,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.507.2" +version = "1.508.0" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ed5d1c7ec5..742ad1374c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.507.2" +version = "1.508.0" authors.workspace = true edition.workspace = true @@ -32,7 +32,7 @@ members = [ ] [workspace.package] -version = "1.507.2" +version = "1.508.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 17f54be854..b3eaddcbc1 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.507.2 + version: 1.508.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 7578655f4c..6b4d6c50ee 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.507.2"; +export const VERSION = "v1.508.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index b298d2a7ad..214a54a566 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -65,7 +65,7 @@ export { // } // }); -export const VERSION = "1.507.2"; +export const VERSION = "1.508.0"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index d93996431f..7f51ee668b 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.507.2", + "version": "1.508.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.507.2", + "version": "1.508.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 8f821a6c7e..0755497117 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.507.2", + "version": "1.508.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 1a98c5d097..0499198817 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.507.2" -wmill_pg = ">=1.507.2" +wmill = ">=1.508.0" +wmill_pg = ">=1.508.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index cf6b5fa467..54c0af034c 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.507.2 + version: 1.508.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index fb7f0db353..08c5fa5274 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.507.2' + ModuleVersion = '1.508.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 61d25dd1a4..d73660430e 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.507.2" +version = "1.508.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 4e0e26fe74..8ce27e3a66 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.507.2" +version = "1.508.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index ab61fd2b7f..93e0c9dd23 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.507.2", + "version": "1.508.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index e19ab5e2a9..92b45520db 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.507.2", + "version": "1.508.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 5f5962f115..28bcf0e708 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.507.2 +1.508.0 From 01d8a3917b6933bb92544536b71808b335f3809b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 17 Jul 2025 07:25:20 +0000 Subject: [PATCH 11/22] fix windows worker toolchain --- .github/workflows/build_windows_worker_.yml | 2 +- .github/workflows/publish_windows_worker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 738b2a9224..446606503d 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -33,7 +33,7 @@ jobs: - name: Setup Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.85.0 + toolchain: 1.88.0 override: true - name: Substitute EE code diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index e9a8c77e87..809e610d58 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -35,7 +35,7 @@ jobs: - name: Setup Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.85.0 + toolchain: 1.88.0 override: true - name: Substitute EE code From 1842157bd543b44c005a95bdcde4e0d329a37461 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Thu, 17 Jul 2025 13:43:09 +0200 Subject: [PATCH 12/22] remove indigo badge (#6210) * remove indigo badge * Revert "remove indigo badge" This reverts commit 843f4460342554e8d7887ede5ebde946756d6333. * fix wrong badge * missing prettifyHeader props in ArgInput / SchemaForm --- frontend/src/lib/components/ArgInput.svelte | 5 +++++ .../src/lib/components/FlowPreviewResult.svelte | 15 +++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 4dc1184eca..1a634b42e2 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -762,6 +762,7 @@ {onlyMaskPassword} {disablePortal} {disabled} + {prettifyHeader} schema={getSchemaFromProperties(itemsType?.properties)} bind:args={value[i]} /> @@ -904,6 +905,7 @@ {onlyMaskPassword} {disablePortal} {disabled} + {prettifyHeader} bind:schema={ () => ({ properties: obj.properties ?? {}, @@ -934,6 +936,7 @@ {onlyMaskPassword} {disablePortal} {disabled} + {prettifyHeader} hiddenArgs={['label', 'kind']} schema={{ properties: obj.properties, @@ -1009,6 +1012,7 @@ {onlyMaskPassword} {disablePortal} {disabled} + {prettifyHeader} bind:schema={ () => ({ properties, @@ -1044,6 +1048,7 @@ {onlyMaskPassword} {disablePortal} {disabled} + {prettifyHeader} schema={{ properties, order, diff --git a/frontend/src/lib/components/FlowPreviewResult.svelte b/frontend/src/lib/components/FlowPreviewResult.svelte index d92192fadb..ceaebe5b15 100644 --- a/frontend/src/lib/components/FlowPreviewResult.svelte +++ b/frontend/src/lib/components/FlowPreviewResult.svelte @@ -1,5 +1,5 @@ - +
{#if scopes == undefined} diff --git a/frontend/src/lib/components/settings/CreateToken.svelte b/frontend/src/lib/components/settings/CreateToken.svelte index fe9287ad52..d4ef7c0b44 100644 --- a/frontend/src/lib/components/settings/CreateToken.svelte +++ b/frontend/src/lib/components/settings/CreateToken.svelte @@ -5,11 +5,12 @@ import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' import Toggle from '../Toggle.svelte' - import { UserService, type NewToken } from '$lib/gen' - import { copyToClipboard } from '$lib/utils' - import { Clipboard } from 'lucide-svelte' - import ClipboardPanel from '../details/ClipboardPanel.svelte' + import { IntegrationService, UserService, type NewToken } from '$lib/gen' import { createEventDispatcher } from 'svelte' + import MultiSelect from '../select/MultiSelect.svelte' + import { safeSelectItems } from '../select/utils.svelte' + import TokenDisplay from './TokenDisplay.svelte' + import ScopeSelector from './ScopeSelector.svelte' interface Props { showMcpMode?: boolean @@ -17,7 +18,7 @@ newTokenLabel?: string defaultNewTokenWorkspace?: string scopes?: string[] - actions?: () => void + onTokenCreated?: (token: string) => void displayCreateToken?: boolean } @@ -25,7 +26,7 @@ showMcpMode = false, defaultNewTokenWorkspace, scopes, - actions, + onTokenCreated, newTokenLabel = $bindable(undefined) }: Props = $props() @@ -38,12 +39,13 @@ let newMcpScope = $state('favorites') let loadingApps = $state(false) let errorFetchApps = $state(false) - // let allApps = $state([]) + let allApps = $state([]) + + let customScopes = $state([]) + let showCustomScopes = $state(false) - function handleCopyClick() { - copyToClipboard(newToken ?? '') - } const dispatch = createEventDispatcher() + function ensureCurrentWorkspaceIncluded( workspacesList: UserWorkspace[], currentWorkspace: string | undefined @@ -71,6 +73,8 @@ if (newMcpApps.length > 0) { tokenScopes.push(`mcp:hub:${newMcpApps.join(',')}`) } + } else if (showCustomScopes && customScopes.length > 0) { + tokenScopes = customScopes } const createdToken = await UserService.createToken({ @@ -89,8 +93,8 @@ } dispatch('tokenCreated', newToken ?? newMcpToken) - if (actions) { - actions() + if (onTokenCreated) { + onTokenCreated(newToken ?? newMcpToken ?? '') } mcpCreationMode = false } catch (err) { @@ -100,37 +104,37 @@ const workspaces = $derived(ensureCurrentWorkspaceIncluded($userWorkspaces, $workspaceStore)) const mcpBaseUrl = $derived(`${window.location.origin}/api/mcp/w/${newTokenWorkspace}/sse?token=`) + + $effect(() => { + if (mcpCreationMode) { + getAllApps() + } else { + newMcpApps = [] + } + }) + + async function getAllApps() { + if (allApps.length > 0) { + return + } + try { + loadingApps = true + allApps = ( + await IntegrationService.listHubIntegrations({ + kind: 'script' + }) + ).map((x) => x.name) + } catch (err) { + console.error('Hub is not available') + allApps = [] + errorFetchApps = true + } finally { + loadingApps = false + } + }
- {#if newToken} -
-
- Added token: -
-
- Make sure to copy your personal access token now. You won't be able to see it again! -
-
- {/if} - - {#if newMcpToken} -
-

New MCP URL:

- -

- Make sure to copy this URL now. You won't be able to see it again! -

-
- {/if} -

Add a new token

@@ -176,7 +180,27 @@
{/if} -
+ {#if !mcpCreationMode && (!scopes || scopes.length === 0)} +
+ { + showCustomScopes = e.detail + }} + options={{ + right: 'Limit token permissions', + rightTooltip: + 'By default, tokens have full API access. Enable this to restrict the token to specific scopes.' + }} + size="xs" + /> + {#if showCustomScopes} + + {/if} +
+ {/if} + +
{#if mcpCreationMode}
Scope @@ -205,7 +229,12 @@ {:else if errorFetchApps}
Error fetching apps
{:else} - + {/if}
@@ -259,4 +288,12 @@
+ + {#if newToken} + + {/if} + + {#if newMcpToken} + + {/if}
diff --git a/frontend/src/lib/components/settings/ScopeSelector.svelte b/frontend/src/lib/components/settings/ScopeSelector.svelte new file mode 100644 index 0000000000..329317518c --- /dev/null +++ b/frontend/src/lib/components/settings/ScopeSelector.svelte @@ -0,0 +1,739 @@ + + +
+ {#if loading} +
+ +
+ {:else if error} +
+

{error}

+ +
+ {:else if scopeDomains} +
+
+

+ Selected Scopes ({selectedScopes.length}) +

+ +
+ + {#if selectedScopes.length === 0} +

No scopes selected. Token will have full access.

+ {:else if hasAdministratorScope} +

Administrator scope grants full access to all resources.

+ {:else} +
+ {#each selectedScopes.slice(0, 10) as scope} + + {scope} + + + {/each} + {#if selectedScopes.length > 10} + + +{selectedScopes.length - 10} more + + {/if} +
+ {/if} +
+ +
+ {#each scopeDomains as domain} + {@const domainState = getDomainState(domain.name)} + {@const isExpanded = domainState?.isExpanded || false} + {@const isDomainSelected = domainState?.hasFullAccess || false} + {@const selectedScopes = getSelectedScopesForDomain(domain)} + +
+ + +
toggleDomainExpansion(domain.name)} + > +
+
+ +
+ +
+ handleDomainCheckboxChange(domain, e.currentTarget.checked)} + onclick={(e) => e.stopPropagation()} + class="!w-4 !h-4 cursor-pointer" + /> +
+ +
+
+ + + {#each selectedScopes as scope} + + {scope} + + + {/each} +
+ {#if domain.description} +

{domain.description}

+ {/if} +
+
+
+ + {#if isExpanded} +
+
+ {#each domain.scopes as scope} + {@const scopeState = domainState?.scopes[scope.value]} + {@const isSelected = scopeState?.isSelected || false} + {@const resourcePathArray = scopeState?.resourcePaths || []} + {@const currentInput = scopeState?.currentInputValue || ''} + {@const pathError = scopeState?.pathError} + {@const isDisabled = disabled || isScopeDisabled(scope, domain)} + +
+
+ +
+ {#if scope.requires_resource_path} + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
+
+ { + if (scopeState) { + scopeState.currentInputValue = e.currentTarget.value + scopeState.pathError = undefined + } + }} + placeholder="e.g. f/folder/*, u/user/path" + onkeydown={(e) => { + if (e.key === 'Enter' && currentInput.trim()) { + e.preventDefault() + addResourcePath(scope.value, currentInput) + } + }} + class="flex-1 text-sm px-3 py-2 border rounded-lg focus:ring-2 focus:ring-blue-500 focus:border-blue-500 bg-surface" + /> + +
+ {#if pathError} +

{pathError}

+ {/if} +
+ {/snippet} + + {/if} +
+
+ + {#if scope.requires_resource_path && resourcePathArray.length > 0} +
+ {#each resourcePathArray as path} + + {path} + + + {/each} +
+ {/if} +
+ {/each} +
+
+ {/if} +
+ {/each} +
+ {/if} +
diff --git a/frontend/src/lib/components/settings/TokenDisplay.svelte b/frontend/src/lib/components/settings/TokenDisplay.svelte new file mode 100644 index 0000000000..cc48b1d2b9 --- /dev/null +++ b/frontend/src/lib/components/settings/TokenDisplay.svelte @@ -0,0 +1,124 @@ + + +
+
+
+ {#if type === 'mcp'} + + + + {:else} + + + + {/if} +
+
+

+ {displayTitle} +

+ + {#if type === 'mcp' && mcpUrl} +
+
+ + + +
+
+ + +
+
+ + {token} + + +
+
+
+ + Make sure to copy both the URL and token now. You won't be able to see them again! + +
+

+ Next steps: Use this URL in your MCP-compatible client (like Claude Desktop) to access your Windmill scripts and flows as tools. +

+
+
+ {:else} +
+
+ + {token} + + +
+
+
+ + Make sure to copy your personal access token now. You won't be able to see it again! + +
+ {/if} +
+
+
\ No newline at end of file diff --git a/frontend/src/lib/components/settings/TokensTable.svelte b/frontend/src/lib/components/settings/TokensTable.svelte index 1fa49cc96b..2056def624 100644 --- a/frontend/src/lib/components/settings/TokensTable.svelte +++ b/frontend/src/lib/components/settings/TokensTable.svelte @@ -1,19 +1,10 @@ -
-

Tokens

-
- +
+
+ Authenticate to the Windmill API with access tokens.
-
-
- Authenticate to the Windmill API with access tokens. -
-
- {#if newToken} -
-
- Added token: -
-
- Make sure to copy your personal access token now. You won't be able to see it again! -
-
- {/if} - - {#if newMcpToken} -
-

New MCP URL:

- -

- Make sure to copy this URL now. You won't be able to see it again! -

-
- {/if} - - {#if displayCreateToken} -
-

Add a new token

- - {#if showMcpMode} -
- { - mcpCreationMode = e.detail - if (e.detail) { - newTokenLabel = 'MCP token' - newTokenExpiration = undefined - newTokenWorkspace = $workspaceStore - } else { - newTokenLabel = undefined - newTokenExpiration = undefined - newTokenWorkspace = defaultNewTokenWorkspace - } - }} - checked={mcpCreationMode} - options={{ - right: 'Generate MCP URL', - rightTooltip: - 'Generate a new MCP URL to make your scripts and flows available as tools through your LLM clients.', - rightDocumentationLink: 'https://www.windmill.dev/docs/core_concepts/mcp' - }} - size="xs" - aiId="account-settings-create-mcp-token" - aiDescription="Create a new MCP token to authenticate to the Windmill API" - /> -
- {/if} - - {#if scopes != undefined} -
- Scope - {#each scopes as scope} - - {/each} -
- {/if} - -
- {#if mcpCreationMode} -
- Scope - - {#snippet children({ item })} - - - {/snippet} - -
- -
- Hub scripts (optional) - {#if loadingApps} -
Loading...
- {:else if errorFetchApps} -
Error fetching apps
- {:else} - - {/if} -
- -
- Workspace - -
- {/if} - -
- Label (optional) - -
- - {#if !mcpCreationMode} -
- Expires In (optional) - -
- {/if} -
- -
- - -
-
- {/if} -
- -
- - - - prefix - label - expiration - scopes - - - {#snippet body()} - - {#if tokens && tokens.length > 0} - {#each tokens as { token_prefix, expiration, label, scopes }} - - {token_prefix}**** - {label ?? ''} - {displayDate(expiration ?? '')} - {scopes?.join(', ') ?? ''} - - - + +
+ + + + prefix + label + expiration + scopes + + + {#snippet body()} + + {#if tokens && tokens.length > 0} + {#each tokens as { token_prefix, expiration, label, scopes }} + + {token_prefix}**** + {label ?? ''} + {displayDate(expiration ?? '')} + {scopes?.join(', ') ?? ''} + + + + + {/each} + {:else if tokens && tokens.length === 0} + + There are no tokens yet - {/each} - {:else if tokens && tokens.length === 0} - - There are no tokens yet - - {:else} - Loading... - {/if} - - {/snippet} - -
- {#if tokens?.length == 100} - - {/if} - {#if tokenPage > 1} - - {/if} + {:else} + Loading... + {/if} + + {/snippet} + +
+ {#if tokens?.length == 100} + + {/if} + {#if tokenPage > 1} + + {/if} +
diff --git a/frontend/src/lib/components/triggers/TriggersWrapper.svelte b/frontend/src/lib/components/triggers/TriggersWrapper.svelte index 31ecb55506..3866e9343a 100644 --- a/frontend/src/lib/components/triggers/TriggersWrapper.svelte +++ b/frontend/src/lib/components/triggers/TriggersWrapper.svelte @@ -68,13 +68,13 @@ {hash} token="" {args} - scopes={isFlow ? [`run:flow/${currentPath}`] : [`run:script/${currentPath}`]} + scopes={isFlow ? [`jobs:run:flows:${currentPath}`] : [`jobs:run:scripts:${currentPath}`]} {newItem} /> {:else if selectedTrigger.type === 'email'}
diff --git a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte index 11b1b3eeb9..f3a0849a7a 100644 --- a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte +++ b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte @@ -64,7 +64,10 @@ } token = undefined - scopes = [`run:${itemKind}/${scriptPath}`] + + + + scopes = [`jobs:run:${itemKind}s:${scriptPath}`] } From e7123ced31038cf722cda67d16b8ad48589bb53a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 17 Jul 2025 16:08:24 +0000 Subject: [PATCH 18/22] fix: prevent idle queries at the sqlx level --- backend/windmill-common/src/lib.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index b619891037..c639d33b0b 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -339,7 +339,7 @@ pub async fn connect( worker_mode: bool, ) -> Result, error::Error> { use std::time::Duration; - + use sqlx::Executor; sqlx::postgres::PgPoolOptions::new() .min_connections((max_connections / 5).clamp(3, max_connections)) .max_connections(max_connections) @@ -347,13 +347,30 @@ pub async fn connect( .after_connect(move |conn, _| { if worker_mode { Box::pin(async move { - sqlx::query("SET enable_seqscan = OFF;") - .execute(conn) - .await?; + if let Err(e) = conn.execute(r#" + SET enable_seqscan = OFF; + SET statement_timeout = '5min'; + SET idle_in_transaction_session_timeout = '10min'; + SET tcp_keepalives_idle = 300; + SET tcp_keepalives_interval = 60; + SET tcp_keepalives_count = 10;"#) + .await { + tracing::error!("Error setting postgres settings: {}", e); + } Ok(()) }) } else { - Box::pin(async move { Ok(()) }) + Box::pin(async move { + if let Err(e) = conn.execute(r#" + SET statement_timeout = '5min'; + SET idle_in_transaction_session_timeout = '10min'; + SET tcp_keepalives_idle = 300; + SET tcp_keepalives_interval = 60; + SET tcp_keepalives_count = 10;"#) + .await { + tracing::error!("Error setting postgres settings: {}", e); + } + Ok(()) }) } }) .connect_with( From 8e87d412acd2bab4fb6c8ce748065657456e33f5 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 17 Jul 2025 18:13:55 -0400 Subject: [PATCH 19/22] fix: fix circular dependancy breaking bundling of cli (#6219) * fix: fix circular dependancy breaking bundling of cli * remove hubpaths --- cli/apps.ts | 3 +- cli/auth.ts | 61 +++++++++++++++++++++++++++++++++++++++++ cli/context.ts | 53 +++++------------------------------ cli/dev.ts | 3 +- cli/flow.ts | 3 +- cli/folder.ts | 3 +- cli/gitsync-settings.ts | 3 +- cli/hub.ts | 3 +- cli/main.ts | 5 ++-- cli/resource-type.ts | 3 +- cli/resource.ts | 3 +- cli/schedule.ts | 3 +- cli/script.ts | 3 +- cli/sync.ts | 3 +- cli/trigger.ts | 5 ++-- cli/user.ts | 2 +- cli/variable.ts | 3 +- cli/workspace.ts | 30 ++++++++++---------- 18 files changed, 112 insertions(+), 80 deletions(-) create mode 100644 cli/auth.ts diff --git a/cli/apps.ts b/cli/apps.ts index ca966e2112..324275d3ed 100644 --- a/cli/apps.ts +++ b/cli/apps.ts @@ -1,5 +1,6 @@ // deno-lint-ignore-file no-explicit-any -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { colors, Command, log, SEP, Table, yamlParseFile } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { ListableApp, Policy } from "./gen/types.gen.ts"; diff --git a/cli/auth.ts b/cli/auth.ts new file mode 100644 index 0000000000..fa50e1f3ec --- /dev/null +++ b/cli/auth.ts @@ -0,0 +1,61 @@ +// deno-lint-ignore-file no-explicit-any +import { colors, log, setClient } from "./deps.ts"; +import * as wmill from "./gen/services.gen.ts"; +import { GlobalUserInfo } from "./gen/types.gen.ts"; + +import { loginInteractive, tryGetLoginInfo } from "./login.ts"; +import { GlobalOptions } from "./types.ts"; +import { + addWorkspace, + removeWorkspace, + Workspace +} from "./workspace.ts"; + +/** + * Main authentication function - moved from context.ts to break circular dependencies + * This function maintains the original API signature from context.ts + */ +export async function requireLogin( + opts: GlobalOptions +): Promise { + // Import resolveWorkspace to avoid circular dependency at module level + const { resolveWorkspace } = await import("./context.ts"); + const workspace = await resolveWorkspace(opts); + + let token = await tryGetLoginInfo(opts); + + if (!token) { + token = workspace.token; + } + + setClient(token, workspace.remote.substring(0, workspace.remote.length - 1)); + + try { + return await wmill.globalWhoami(); + } catch (error) { + // Check for network errors and provide clearer messages + const errorMsg = error instanceof Error ? error.message : String(error); + if (errorMsg.includes('fetch') || errorMsg.includes('connection') || errorMsg.includes('ECONNREFUSED') || errorMsg.includes('refused')) { + throw new Error(`Network error: Could not connect to Windmill server at ${workspace.remote}`); + } + + log.info( + "! Could not reach API given existing credentials. Attempting to reauth..." + ); + const newToken = await loginInteractive(workspace.remote); + if (!newToken) { + throw new Error("Unauthorized: Could not authenticate with the provided credentials"); + } + + // Update workspace token + removeWorkspace(workspace.name, false, opts); + workspace.token = newToken; + addWorkspace(workspace, opts); + + setClient( + newToken, + workspace.remote.substring(0, workspace.remote.length - 1) + ); + return await wmill.globalWhoami(); + } +} diff --git a/cli/context.ts b/cli/context.ts index c5fd7a4571..359854c880 100644 --- a/cli/context.ts +++ b/cli/context.ts @@ -58,12 +58,12 @@ export async function resolveWorkspace( if (opts.baseUrl) { if (opts.workspace && opts.token) { const normalizedBaseUrl = new URL(opts.baseUrl).toString(); // add trailing slash if not present - + // Try to find existing workspace profile by name, then by workspaceId + remote if (opts.workspace) { // Try by workspace name first let existingWorkspace = await getWorkspaceByName(opts.workspace, opts.configDir); - + // If not found by name, try to find by workspaceId + remote match if (!existingWorkspace) { const { allWorkspaces } = await import("./workspace.ts"); @@ -71,13 +71,13 @@ export async function resolveWorkspace( const matchingWorkspaces = workspaces.filter( w => w.workspaceId === opts.workspace && w.remote === normalizedBaseUrl ); - + // Due to uniqueness constraint, there can only be 0 or 1 match if (matchingWorkspaces.length === 1) { existingWorkspace = matchingWorkspaces[0]; } } - + if (existingWorkspace) { // Validate that the base URL matches the profile's remote if (existingWorkspace.remote !== normalizedBaseUrl) { @@ -95,7 +95,7 @@ export async function resolveWorkspace( }; } } - + // No existing profile found, create temporary workspace return { remote: normalizedBaseUrl, @@ -121,45 +121,6 @@ export async function resolveWorkspace( } } -export async function requireLogin( - opts: GlobalOptions -): Promise { - const workspace = await resolveWorkspace(opts); - let token = await tryGetLoginInfo(opts); - - if (!token) { - token = workspace.token; - } - - setClient(token, workspace.remote.substring(0, workspace.remote.length - 1)); - - try { - return await wmill.globalWhoami(); - } catch (error) { - // Check for network errors and provide clearer messages - const errorMsg = error instanceof Error ? error.message : String(error); - if (errorMsg.includes('fetch') || errorMsg.includes('connection') || errorMsg.includes('ECONNREFUSED') || errorMsg.includes('refused')) { - throw new Error(`Network error: Could not connect to Windmill server at ${workspace.remote}`); - } - - log.info( - "! Could not reach API given existing credentials. Attempting to reauth..." - ); - const newToken = await loginInteractive(workspace.remote); - if (!newToken) { - throw new Error("Unauthorized: Could not authenticate with the provided credentials"); - } - removeWorkspace(workspace.name, false, opts); - workspace.token = newToken; - addWorkspace(workspace, opts); - - setClient( - newToken, - workspace.remote.substring(0, workspace.remote.length - 1) - ); - return await wmill.globalWhoami(); - } -} export async function fetchVersion(baseUrl: string): Promise { const requestHeaders = new Headers(); @@ -176,13 +137,13 @@ export async function fetchVersion(baseUrl: string): Promise { new URL(new URL(baseUrl).origin + "/api/version"), { headers: requestHeaders, method: "GET" } ); - + if (!response.ok) { // Consume response body even on error to avoid resource leak await response.text(); throw new Error(`Failed to fetch version: ${response.status} ${response.statusText}`); } - + return await response.text(); } export async function tryResolveVersion( diff --git a/cli/dev.ts b/cli/dev.ts index 7cc92c45ff..658a1a057a 100644 --- a/cli/dev.ts +++ b/cli/dev.ts @@ -12,7 +12,8 @@ import { } from "./deps.ts"; import { getTypeStrFromPath, GlobalOptions } from "./types.ts"; import { ignoreF } from "./sync.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace } from "./context.ts"; import { SyncOptions, mergeConfigWithConfigFile, diff --git a/cli/flow.ts b/cli/flow.ts index 8b44011b06..bf3acd67ad 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -4,7 +4,8 @@ import { Confirm, SEP, log, yamlStringify } from "./deps.ts"; import { colors, Command, Table, yamlParseFile } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { resolve, track_job } from "./script.ts"; import { defaultFlowDefinition } from "./bootstrap/flow_bootstrap.ts"; import { blueColor, generateFlowLockInternal } from "./metadata.ts"; diff --git a/cli/folder.ts b/cli/folder.ts index 5cf2812670..e60dc08903 100644 --- a/cli/folder.ts +++ b/cli/folder.ts @@ -2,7 +2,8 @@ import { colors, Command, log, SEP, Table } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts"; import { Folder } from "./gen/types.gen.ts"; diff --git a/cli/gitsync-settings.ts b/cli/gitsync-settings.ts index 3daeaa0415..d48105c9f3 100644 --- a/cli/gitsync-settings.ts +++ b/cli/gitsync-settings.ts @@ -1,6 +1,7 @@ import { colors, Command, log, yamlStringify } from "./deps.ts"; import { GlobalOptions } from "./types.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace } from "./context.ts"; import * as wmill from "./gen/services.gen.ts"; import { DEFAULT_SYNC_OPTIONS, diff --git a/cli/hub.ts b/cli/hub.ts index 6943369ec0..2f0eb0c567 100644 --- a/cli/hub.ts +++ b/cli/hub.ts @@ -2,7 +2,8 @@ import { Command, log } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace } from "./context.ts"; import { pushResourceType } from "./resource-type.ts"; import { GlobalOptions } from "./types.ts"; import { deepEqual } from "./utils.ts"; diff --git a/cli/main.ts b/cli/main.ts index 214a54a566..df61a212e2 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -159,9 +159,8 @@ const command = new Command() // Check for backend git-sync settings unless --use-default is specified if (!opts.useDefault) { try { - const { requireLogin, resolveWorkspace } = await import( - "./context.ts" - ); + const { requireLogin } = await import("./auth.ts"); + const { resolveWorkspace } = await import("./context.ts"); // Check if user has workspace configured const { getActiveWorkspace } = await import( diff --git a/cli/resource-type.ts b/cli/resource-type.ts index 7a25c5a36e..aef94d2249 100644 --- a/cli/resource-type.ts +++ b/cli/resource-type.ts @@ -5,7 +5,8 @@ import { parseFromFile, removeType, } from "./types.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace } from "./context.ts"; import { colors, Command, log, Table } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { ResourceType } from "./gen/types.gen.ts"; diff --git a/cli/resource.ts b/cli/resource.ts index 1b65cf9420..0d620f68e3 100644 --- a/cli/resource.ts +++ b/cli/resource.ts @@ -5,7 +5,8 @@ import { parseFromFile, removeType, } from "./types.ts"; -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { colors, Command, log, SEP, Table } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { Resource } from "./gen/types.gen.ts"; diff --git a/cli/schedule.ts b/cli/schedule.ts index 77052e371d..34ec378a18 100644 --- a/cli/schedule.ts +++ b/cli/schedule.ts @@ -1,6 +1,7 @@ // deno-lint-ignore-file no-explicit-any import { colors, Command, log, SEP, Table } from "./deps.ts"; -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import * as wmill from "./gen/services.gen.ts"; import { diff --git a/cli/script.ts b/cli/script.ts index b5d978e13f..8ae5e3f6e6 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -1,6 +1,7 @@ // deno-lint-ignore-file no-explicit-any import { GlobalOptions } from "./types.ts"; -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { colors, Command, diff --git a/cli/sync.ts b/cli/sync.ts index 2fe403e2c5..162c48282e 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -1,4 +1,5 @@ -import { fetchVersion, requireLogin, resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { fetchVersion, resolveWorkspace } from "./context.ts"; import { colors, Command, diff --git a/cli/trigger.ts b/cli/trigger.ts index 2edd6b1674..df21755794 100644 --- a/cli/trigger.ts +++ b/cli/trigger.ts @@ -16,9 +16,8 @@ import { parseFromFile, removeType, } from "./types.ts"; -import { requireLogin } from "./context.ts"; -import { validatePath } from "./context.ts"; -import { resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { validatePath, resolveWorkspace } from "./context.ts"; type Trigger = { http: HttpTrigger; diff --git a/cli/user.ts b/cli/user.ts index 70e069f303..668f294d67 100644 --- a/cli/user.ts +++ b/cli/user.ts @@ -1,5 +1,5 @@ // deno-lint-ignore-file no-explicit-any -import { requireLogin } from "./context.ts"; +import { requireLogin } from "./auth.ts"; import { GlobalOptions, isSuperset, diff --git a/cli/variable.ts b/cli/variable.ts index 789334c44c..8524959c32 100644 --- a/cli/variable.ts +++ b/cli/variable.ts @@ -1,5 +1,6 @@ // deno-lint-ignore-file no-explicit-any -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { GlobalOptions, isSuperset, diff --git a/cli/workspace.ts b/cli/workspace.ts index b2af05d089..9c0b865a00 100644 --- a/cli/workspace.ts +++ b/cli/workspace.ts @@ -3,9 +3,9 @@ import { GlobalOptions } from "./types.ts"; import { getRootStore } from "./store.ts"; import { loginInteractive, tryGetLoginInfo } from "./login.ts"; import { colors, Command, Confirm, Input, log, setClient, Table } from "./deps.ts"; +import { requireLogin } from "./auth.ts"; import * as wmill from "./gen/services.gen.ts"; -import { requireLogin } from "./context.ts"; export interface Workspace { remote: string; @@ -258,11 +258,11 @@ export async function add( export async function addWorkspace(workspace: Workspace, opts: any) { workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases! - + // Check for conflicts before adding const existingWorkspaces = await allWorkspaces(opts.configDir); const isInteractive = Deno.stdin.isTerminal() && Deno.stdout.isTerminal() && !opts.force; - + // Check 1: Workspace name already exists const nameConflict = existingWorkspaces.find(w => w.name === workspace.name); if (nameConflict) { @@ -274,7 +274,7 @@ export async function addWorkspace(workspace: Workspace, opts: any) { log.info(colors.red.bold(`❌ Workspace name "${workspace.name}" already exists!`)); log.info(` Existing: ${nameConflict.workspaceId} on ${nameConflict.remote}`); log.info(` New: ${workspace.workspaceId} on ${workspace.remote}`); - + if (!isInteractive) { // In non-interactive mode (tests, scripts), auto-overwrite with force flag if (opts.force) { @@ -287,7 +287,7 @@ export async function addWorkspace(workspace: Workspace, opts: any) { message: "Do you want to overwrite the existing workspace?", default: false, }); - + if (!overwrite) { log.info(colors.yellow("Operation cancelled.")); return; @@ -295,20 +295,20 @@ export async function addWorkspace(workspace: Workspace, opts: any) { } } } - + // Check 2: Same (remote, workspaceId) tuple already exists under different name - const tupleConflict = existingWorkspaces.find(w => - w.remote === workspace.remote && - w.workspaceId === workspace.workspaceId && + const tupleConflict = existingWorkspaces.find(w => + w.remote === workspace.remote && + w.workspaceId === workspace.workspaceId && w.name !== workspace.name ); - + if (tupleConflict) { log.info(colors.red.bold(`❌ Workspace ${workspace.workspaceId} on ${workspace.remote} already exists!`)); log.info(` Existing name: "${tupleConflict.name}"`); log.info(` New name: "${workspace.name}"`); log.info(colors.yellow(`\nNote: Backend constraint prevents duplicate (remote, workspaceId) combinations.`)); - + if (!isInteractive) { // In non-interactive mode (tests, scripts), auto-overwrite with force flag if (opts.force) { @@ -321,20 +321,20 @@ export async function addWorkspace(workspace: Workspace, opts: any) { message: `Do you want to overwrite the existing workspace "${tupleConflict.name}"?`, default: false, }); - + if (!overwrite) { log.info(colors.yellow("Operation cancelled.")); return; } } - + // Remove the conflicting workspace await removeWorkspace(tupleConflict.name, true, opts); } - + // Remove existing workspace with same name (if updating) await removeWorkspace(workspace.name, true, opts); - + // Add the new workspace const file = await Deno.open((await getRootStore(opts.configDir)) + "remotes.ndjson", { append: true, From a2d9438b92831bf80ed671dd04b19df30b5daa8a Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 18 Jul 2025 00:14:09 +0200 Subject: [PATCH 20/22] chore: upgrade bun to 1.2.18 (#6218) --- .github/DockerfileBackendTests | 2 +- Dockerfile | 2 +- backend/windmill-worker/loader_builder.bun.js | 126 +++++++++--------- backend/windmill-worker/src/bun_executor.rs | 52 ++++---- docker/DockerfileSlim | 2 +- docker/DockerfileSlimEe | 2 +- 6 files changed, 92 insertions(+), 94 deletions(-) diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 7fda0e025f..dd8452e2cf 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -42,7 +42,7 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER RUN /usr/local/bin/python3 -m pip install pip-tools # Bun -COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun ARG TARGETPLATFORM diff --git a/Dockerfile b/Dockerfile index 934ef10148..3815744d1e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -194,7 +194,7 @@ COPY --from=builder /windmill/target/release/windmill ${APP}/windmill COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno -COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/backend/windmill-worker/loader_builder.bun.js b/backend/windmill-worker/loader_builder.bun.js index dec0e41fed..2bf0b749f2 100644 --- a/backend/windmill-worker/loader_builder.bun.js +++ b/backend/windmill-worker/loader_builder.bun.js @@ -1,9 +1,15 @@ -const bo = await Bun.build({ - entrypoints: ["./main.ts"], - outdir: "./out", - plugins: [p], - external: ["*"], -}); +try { + await Bun.build({ + entrypoints: ["./main.ts"], + outdir: "./out", + plugins: [p], + external: ["*"], + }); +} catch (err) { + console.log(err); + console.log("Failed to build bundle"); + process.exit(1); +} const fs = require("fs/promises"); @@ -12,66 +18,58 @@ const captureVersion = import { semver } from "bun"; -if (!bo.success) { - bo.logs.forEach((l) => console.log(l)); - console.log("Failed to build bundle"); - process.exit(1); -} else { - let content = await fs.readFile("./out/main.js", { encoding: "utf8" }); - const imports = new Bun.Transpiler().scanImports( - content.replaceAll("__require", "require") - ); +let content = await fs.readFile("./out/main.js", { encoding: "utf8" }); +const imports = new Bun.Transpiler().scanImports( + content.replaceAll("__require", "require") +); - const dependencies = {}; - for (const i of imports) { - let [_, name, version] = i.path.match(captureVersion) ?? []; - if (name == undefined) { - throw Error("Unrecognized import: " + i.path); - } - if (name.startsWith("node:")) { - continue; - } - let splitted = name.split("/"); - if (splitted.length > 2) { - name = splitted.slice(0, 2).join("/"); - } - if (version == undefined) { - if (dependencies[name] == undefined) { - dependencies[name] = []; - } - } else { - if (dependencies[name] == undefined) { - dependencies[name] = [version]; - } else if (!dependencies[name].includes(version)) { - dependencies[name].push(version); - } - } +const dependencies = {}; +for (const i of imports) { + let [_, name, version] = i.path.match(captureVersion) ?? []; + if (name == undefined) { + throw Error("Unrecognized import: " + i.path); } - const resolvedDeps = {}; - for (const i in dependencies) { - const versions = dependencies[i]; - resolvedDeps[i] = - versions.length == 0 - ? "latest" - : versions.length == 1 - ? versions[0] - : reduceIntersect(versions, i); + if (name.startsWith("node:")) { + continue; } - await Bun.write( - "./package.json", - JSON.stringify({ dependencies: resolvedDeps }, null, 2) - ); - - function reduceIntersect(versions, name) { - console.log( - `multiple versions detected for ${name}: ${versions.join(", ")}` - ); - const regex = /^(?:\^|~|<|<=|>=|>)+/g; - - const r = versions - .map((x) => [x, x.replace(regex, "")]) - .sort((a, b) => semver.order(a[1], b[1]))[0][0]; - console.log(`resolved to ${r} for ${name}`); - return r; + let splitted = name.split("/"); + if (splitted.length > 2) { + name = splitted.slice(0, 2).join("/"); + } + if (version == undefined) { + if (dependencies[name] == undefined) { + dependencies[name] = []; + } + } else { + if (dependencies[name] == undefined) { + dependencies[name] = [version]; + } else if (!dependencies[name].includes(version)) { + dependencies[name].push(version); + } } } +const resolvedDeps = {}; +for (const i in dependencies) { + const versions = dependencies[i]; + resolvedDeps[i] = + versions.length == 0 + ? "latest" + : versions.length == 1 + ? versions[0] + : reduceIntersect(versions, i); +} +await Bun.write( + "./package.json", + JSON.stringify({ dependencies: resolvedDeps }, null, 2) +); + +function reduceIntersect(versions, name) { + console.log(`multiple versions detected for ${name}: ${versions.join(", ")}`); + const regex = /^(?:\^|~|<|<=|>=|>)+/g; + + const r = versions + .map((x) => [x, x.replace(regex, "")]) + .sort((a, b) => semver.order(a[1], b[1]))[0][0]; + console.log(`resolved to ${r} for ${name}`); + return r; +} diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index f724f564df..a508f3084c 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -418,17 +418,17 @@ try {{ }} -const bo = await Bun.build({{ - entrypoints: ["{job_dir}/wrapper.mjs"], - outdir: "./", - target: "node", - plugins: [p], - external: fileNames, - minify: true, - }}); - -if (!bo.success) {{ - bo.logs.forEach((l) => console.log(l)); +try {{ + await Bun.build({{ + entrypoints: ["{job_dir}/wrapper.mjs"], + outdir: "./", + target: "node", + plugins: [p], + external: fileNames, + minify: true, + }}); +}} catch(err) {{ + console.log(err); console.log("Failed to build node bundle"); process.exit(1); }} @@ -462,21 +462,21 @@ plugin(p) r#" {} -const bo = await Bun.build({{ - entrypoints: ["{job_dir}/main.ts"], - outdir: "./", - target: "{}", - plugins: [p], - external: ["electron"], - minify: {{ - identifiers: false, - syntax: true, - whitespace: false - }}, - }}); - -if (!bo.success) {{ - bo.logs.forEach((l) => console.log(l)); +try {{ + await Bun.build({{ + entrypoints: ["{job_dir}/main.ts"], + outdir: "./", + target: "{}", + plugins: [p], + external: ["electron"], + minify: {{ + identifiers: false, + syntax: true, + whitespace: false + }}, + }}); +}} catch(err) {{ + console.log(err) console.log("Failed to build node bundle"); process.exit(1); }} diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 3be8c6c5b8..ca749b02f0 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index f3f1e26cee..4b001f5d10 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ From ec1ed0ba6b12c80879554e8b394870051448c65d Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 17 Jul 2025 18:14:40 -0400 Subject: [PATCH 21/22] cli: git sync improvements v2 (#6220) * log when override is applied vs default taken in git sync * simplify cli merging options + add explicit override test * gitsync-settings pull/push ask for confirmation or --yes if tty * cli legacy backend repo setting detected + interactive migration --- cli/conf.ts | 26 +- cli/gitsync-settings.ts | 336 +++++++++++++++++---- cli/main.ts | 1 + cli/sync.ts | 43 +-- cli/test/gitsync_settings_features.test.ts | 16 +- cli/test/init_no_git_sync.test.ts | 6 +- cli/test/multi_instance_workspace.test.ts | 60 ++-- cli/test/sync_config_resolution.test.ts | 108 +++++-- cli/utils.ts | 4 +- 9 files changed, 433 insertions(+), 167 deletions(-) diff --git a/cli/conf.ts b/cli/conf.ts index 342eb423af..6c4e37e3c8 100644 --- a/cli/conf.ts +++ b/cli/conf.ts @@ -108,31 +108,45 @@ export function getEffectiveSettings( ): SyncOptions { // Start with empty object - no defaults let effective = {} as SyncOptions; - + // Merge top-level settings from config (which contains user's chosen defaults) Object.keys(config).forEach(key => { if (key !== 'overrides' && config[key as keyof SyncOptions] !== undefined) { (effective as any)[key] = config[key as keyof SyncOptions]; } }); - + if (!config.overrides) { + if (repo) { + log.info(`No overrides found in wmill.yaml, using top-level settings (repository flag ignored)`); + } return effective; } - + // Construct override keys using the single format const workspaceKey = `${baseUrl}:${workspaceId}:*`; const repoKey = `${baseUrl}:${workspaceId}:${repo}`; - + + let appliedOverrides: string[] = []; + // Apply workspace-level overrides if (config.overrides[workspaceKey]) { Object.assign(effective, config.overrides[workspaceKey]); + appliedOverrides.push("workspace-level"); } - + // Apply repository-specific overrides (overrides workspace-level) if (config.overrides[repoKey]) { Object.assign(effective, config.overrides[repoKey]); + appliedOverrides.push("repository-specific"); + } else if (repo) { + // Repository was specified but no override found + log.info(`Repository override not found for "${repo}", using ${appliedOverrides.length > 0 ? appliedOverrides.join(" + ") : "top-level"} settings`); } - + + if (appliedOverrides.length > 0) { + log.info(`Applied ${appliedOverrides.join(" + ")} overrides${repo ? ` for repository "${repo}"` : ""}`); + } + return effective; } diff --git a/cli/gitsync-settings.ts b/cli/gitsync-settings.ts index d48105c9f3..27de3b7056 100644 --- a/cli/gitsync-settings.ts +++ b/cli/gitsync-settings.ts @@ -1,4 +1,4 @@ -import { colors, Command, log, yamlStringify } from "./deps.ts"; +import { colors, Command, Confirm, log, yamlStringify } from "./deps.ts"; import { GlobalOptions } from "./types.ts"; import { requireLogin } from "./auth.ts"; import { resolveWorkspace } from "./context.ts"; @@ -164,6 +164,147 @@ function includeTypeToSyncOptions( }; } +// Shared migration function for legacy repositories +async function handleLegacyRepositoryMigration( + selectedRepo: any, + gitSyncSettings: any, + workspace: any, + opts: { yes?: boolean }, + operationName: string = "operation" +): Promise { + if (selectedRepo.settings) { + return selectedRepo; // Already migrated + } + + // This repository is in legacy format - handle migration + if (!gitSyncSettings.include_path || !gitSyncSettings.include_type) { + throw new Error( + `Repository "${selectedRepo.git_repo_resource_path}" has legacy format but workspace-level include_path or include_type is missing. This indicates corrupted git-sync settings.` + ); + } + + const workspaceIncludePath = gitSyncSettings.include_path; + const workspaceIncludeType = gitSyncSettings.include_type; + + if (Deno.stdout.isTerminal() && !opts.yes) { + // Interactive mode - show migration prompt + console.log(colors.yellow('\n⚠️ Legacy git-sync settings detected!')); + console.log(`\nRepository "${selectedRepo.git_repo_resource_path}" has legacy settings format.`); + console.log('The new format allows per-repository filter configuration.'); + if (operationName === "push") { + console.log('This repository must be migrated before pushing settings.\n'); + } else { + console.log('\n'); + } + + console.log(colors.bold('Current workspace-level settings:')); + console.log(` Include paths: ${workspaceIncludePath.join(', ')}`); + console.log(` Include types: ${workspaceIncludeType.join(', ')}\n`); + + // Show what the migration will do + let finalIncludeType = [...workspaceIncludeType]; + if (selectedRepo.exclude_types_override && selectedRepo.exclude_types_override.length > 0) { + const originalCount = finalIncludeType.length; + finalIncludeType = finalIncludeType.filter( + type => !selectedRepo.exclude_types_override.includes(type) + ); + const excludedCount = originalCount - finalIncludeType.length; + console.log(colors.yellow(`Repository excludes ${excludedCount} types: ${selectedRepo.exclude_types_override.join(', ')}`)); + } + + console.log(colors.bold('\nAfter migration, repository will have:')); + console.log(` Include paths: ${workspaceIncludePath.join(', ')}`); + console.log(` Include types: ${finalIncludeType.join(', ')}\n`); + + const confirm = await Confirm.prompt({ + message: operationName === "push" + ? 'Do you want to migrate this repository before pushing?' + : 'Do you want to migrate this repository?', + default: true + }); + + if (!confirm) { + const message = operationName === "push" + ? '\n⚠️ Migration skipped. Cannot push to legacy repository.' + : '\n⚠️ Migration skipped. You can migrate later via the UI.'; + console.log(colors.yellow(message)); + if (operationName === "push") { + return null; // Signal to exit push operation + } + throw new Error('Migration cancelled by user'); + } + + // Perform the migration + let migratedIncludeType = [...workspaceIncludeType]; + if (selectedRepo.exclude_types_override && selectedRepo.exclude_types_override.length > 0) { + migratedIncludeType = migratedIncludeType.filter( + type => !selectedRepo.exclude_types_override.includes(type) + ); + } + + const migratedRepo = { + ...selectedRepo, + settings: { + include_path: [...workspaceIncludePath], + include_type: migratedIncludeType, + exclude_path: [], + extra_include_path: [] + } + }; + + // Remove the old field + delete migratedRepo.exclude_types_override; + + // Update the backend with migrated repository + const updatedRepositories = gitSyncSettings.repositories.map((repo: any) => { + if (repo.git_repo_resource_path === selectedRepo.git_repo_resource_path) { + return migratedRepo; + } + return repo; + }); + + await wmill.editWorkspaceGitSyncConfig({ + workspace: workspace.workspaceId, + requestBody: { + git_sync_settings: { + repositories: updatedRepositories, + // Keep workspace-level settings if other repos are still legacy + ...(gitSyncSettings.repositories.some((r: any) => r.git_repo_resource_path !== selectedRepo.git_repo_resource_path && !r.settings) && { + include_path: workspaceIncludePath, + include_type: workspaceIncludeType + }) + } + } + }); + + console.log(colors.green('\n✓ Repository migration completed successfully!')); + if (operationName === "push") { + console.log('Now proceeding with push operation...\n'); + } + return migratedRepo; + + } else { + // Non-interactive mode - show error + console.error(colors.red('\n❌ Legacy git-sync settings detected!')); + console.error(`\nRepository "${selectedRepo.git_repo_resource_path}" has legacy settings format.`); + if (operationName === "push") { + console.error('This repository must be migrated before pushing settings.'); + } + console.error('Please choose one of the following options:\n'); + console.error('1. Go to the Windmill UI > Workspace Settings > Git Sync'); + console.error(' Review and save this repository to migrate to the new format.\n'); + console.error('2. Run this command in interactive mode (with TTY) to migrate.'); + console.error(` Example: wmill gitsync-settings ${operationName}\n`); + if (operationName === "push") { + console.error('3. Pull settings first to migrate: wmill gitsync-settings pull\n'); + } else { + console.error('3. Push local settings to override backend settings:'); + console.error(' wmill gitsync-settings push\n'); + } + Deno.exit(1); + } +} + // Convert SyncOptions boolean flags to backend include_type array function syncOptionsToIncludeType(opts: SyncOptions): string[] { const includeTypes: string[] = []; @@ -279,6 +420,31 @@ function displayChanges( } } +async function selectAndLogRepository( + repositories: GitSyncRepository[], + repository?: string, +): Promise { + let selectedRepo: GitSyncRepository; + + if (repository) { + const found = repositories.find( + (r: GitSyncRepository) => + r.git_repo_resource_path === repository || + r.git_repo_resource_path === `$res:${repository}`, + ); + if (!found) { + throw new Error(`Repository ${repository} not found`); + } + selectedRepo = found; + const repoPath = selectedRepo.git_repo_resource_path.replace(/^\$res:/, ""); + log.info(colors.cyan(`Using repository: ${colors.bold(repoPath)}`)); + } else { + selectedRepo = await selectRepository(repositories); + } + + return selectedRepo; +} + async function pullGitSyncSettings( opts: GlobalOptions & { repository?: string; @@ -289,6 +455,7 @@ async function pullGitSyncSettings( replace?: boolean; override?: boolean; withBackendSettings?: string; + yes?: boolean; }, ) { const workspace = await resolveWorkspace(opts); @@ -401,22 +568,19 @@ async function pullGitSyncSettings( } // Find the repository to work with - let selectedRepo: GitSyncRepository; - if (opts.repository) { - const found = settings.git_sync.repositories.find( - (r: GitSyncRepository) => - r.git_repo_resource_path === opts.repository || - r.git_repo_resource_path === `$res:${opts.repository}`, - ); - if (!found) { - throw new Error(`Repository ${opts.repository} not found`); - } - selectedRepo = found; - } else { - selectedRepo = await selectRepository( - settings.git_sync.repositories, - ); - } + let selectedRepo = await selectAndLogRepository( + settings.git_sync.repositories, + opts.repository, + ); + + // Check if the selected repository needs migration and handle it + selectedRepo = await handleLegacyRepositoryMigration( + selectedRepo, + settings.git_sync, + workspace, + opts, + "pull" + ); // Convert backend settings to SyncOptions format const backendSyncOptions: SyncOptions = { @@ -534,7 +698,7 @@ async function pullGitSyncSettings( ); } else { if (hasChanges) { - log.info("Changes that would be made:"); + log.info("Changes that would be applied locally:"); const changes = generateChanges(normalizedCurrent, normalizedBackend); if (Object.keys(changes).length === 0) { @@ -573,7 +737,14 @@ async function pullGitSyncSettings( ); const hasConflict = !deepEqual(gitSyncBackend, gitSyncCurrent); - if (hasConflict && Deno.stdin.isTerminal()) { + if (hasConflict && !opts.yes && Deno.stdin.isTerminal()) { + // Show the diff first + log.info("Changes that would be applied locally:"); + const changes = generateChanges(currentSettings, backendSyncOptions); + if (Object.keys(changes).length > 0) { + displayChanges(changes); + } + // Interactive mode - ask user const { Select } = await import("./deps.ts"); const choice = await Select.prompt({ @@ -607,6 +778,16 @@ async function pullGitSyncSettings( repoPath, ); } + } else if (hasConflict && opts.yes) { + // --yes flag: default to override behavior for conflicts + writeMode = "override"; + const repoPath = normalizeRepoPath(selectedRepo.git_repo_resource_path); + overrideKey = constructOverrideKey( + workspace.remote, + workspace.workspaceId, + repoPath, + ); + log.info(colors.yellow("Settings conflict detected. Using --override behavior (default for --yes).")); } else if (hasConflict) { // Non-interactive mode with conflicts - show message and exit if (opts.jsonOutput) { @@ -664,11 +845,19 @@ async function pullGitSyncSettings( // Apply the settings based on write mode let updatedConfig: SyncOptions; + // Log which settings mode is being used + const repoPath = selectedRepo.git_repo_resource_path.replace(/^\$res:/, ""); + if (writeMode === "override") { + log.info(`Applied repository-specific overrides for repository "${repoPath}"`); + } else { + log.info(`Applied settings for repository "${repoPath}"`); + } + if (writeMode === "replace") { // Preserve existing local config and update only git-sync fields updatedConfig = { ...localConfig }; - // Remove overrides since we're in replace mode - delete updatedConfig.overrides; + // Clear overrides since we're in replace mode, but keep empty object for consistency + updatedConfig.overrides = {}; // Update with backend git-sync settings Object.assign(updatedConfig, backendSyncOptions); } else if (writeMode === "override" && overrideKey) { @@ -761,6 +950,7 @@ async function pushGitSyncSettings( diff?: boolean; jsonOutput?: boolean; withBackendSettings?: string; + yes?: boolean; }, ) { const workspace = await resolveWorkspace(opts); @@ -888,21 +1078,23 @@ async function pushGitSyncSettings( } // Find the repository to work with - let selectedRepo: GitSyncRepository; - if (opts.repository) { - const found = settings.git_sync.repositories.find( - (r: GitSyncRepository) => - r.git_repo_resource_path === opts.repository || - r.git_repo_resource_path === `$res:${opts.repository}`, - ); - if (!found) { - throw new Error(`Repository ${opts.repository} not found`); - } - selectedRepo = found; - } else { - selectedRepo = await selectRepository( - settings.git_sync.repositories, - ); + let selectedRepo = await selectAndLogRepository( + settings.git_sync.repositories, + opts.repository, + ); + + // Check if the selected repository needs migration and handle it + selectedRepo = await handleLegacyRepositoryMigration( + selectedRepo, + settings.git_sync, + workspace, + opts, + "push" + ); + + // If migration was cancelled, exit + if (selectedRepo === null) { + return; } // Get effective settings for this workspace/repo @@ -922,24 +1114,25 @@ async function pushGitSyncSettings( extra_include_path: effectiveSettings.extraIncludes || [], }; + // Calculate diff for all modes + const currentBackend = selectedRepo.settings; + + // Convert current backend settings to SyncOptions for user-friendly display + const currentSyncOptions: SyncOptions = { + includes: currentBackend.include_path || [], + excludes: currentBackend.exclude_path || [], + extraIncludes: currentBackend.extra_include_path || [], + ...includeTypeToSyncOptions(currentBackend.include_type || []), + }; + + const normalizedCurrent = normalizeSyncOptions(currentSyncOptions); + const normalizedEffective = normalizeSyncOptions(effectiveSettings); + const gitSyncCurrent = extractGitSyncFields(normalizedCurrent); + const gitSyncEffective = extractGitSyncFields(normalizedEffective); + const hasChanges = !deepEqual(gitSyncEffective, gitSyncCurrent); + if (opts.diff) { - // Show what would be pushed - const currentBackend = selectedRepo.settings; - - // Convert current backend settings to SyncOptions for user-friendly display - const currentSyncOptions: SyncOptions = { - includes: currentBackend.include_path || [], - excludes: currentBackend.exclude_path || [], - extraIncludes: currentBackend.extra_include_path || [], - ...includeTypeToSyncOptions(currentBackend.include_type || []), - }; - - const normalizedCurrent = normalizeSyncOptions(currentSyncOptions); - const normalizedEffective = normalizeSyncOptions(effectiveSettings); - const gitSyncCurrent = extractGitSyncFields(normalizedCurrent); - const gitSyncEffective = extractGitSyncFields(normalizedEffective); - const hasChanges = !deepEqual(gitSyncEffective, gitSyncCurrent); - + // --diff flag: show differences and exit if (opts.jsonOutput) { // Generate structured diff using the same normalized objects const structuredDiff = hasChanges @@ -958,7 +1151,7 @@ async function pushGitSyncSettings( ); } else { if (hasChanges) { - log.info("Changes that would be pushed:"); + log.info("Changes that would be pushed to Windmill:"); const changes = generateChanges( normalizedCurrent, normalizedEffective, @@ -976,6 +1169,39 @@ async function pushGitSyncSettings( return; } + // Default behavior: show changes and ask for confirmation (unless --yes is passed) + if (hasChanges) { + if (!opts.jsonOutput) { + const changes = generateChanges( + normalizedCurrent, + normalizedEffective, + ); + + if (Object.keys(changes).length === 0) { + log.info(colors.green("No changes to push")); + return; + } else { + log.info("Changes that would be pushed to Windmill:"); + displayChanges(changes); + } + } + + // Ask for confirmation unless --yes is passed or not in TTY + if (!opts.yes && Deno.stdin.isTerminal()) { + const confirmed = await Confirm.prompt({ + message: `Do you want to apply these changes to the remote?`, + default: true, + }); + + if (!confirmed) { + return; + } + } + } else { + log.info(colors.green("No changes to push")); + return; + } + if (opts.withBackendSettings) { // Skip backend update when using simulated settings if (opts.jsonOutput) { @@ -1104,6 +1330,7 @@ const command = new Command() "--with-backend-settings ", "Use provided JSON settings instead of querying backend (for testing)", ) + .option("--yes", "Skip interactive prompts and use default behavior") .action(pullGitSyncSettings as any) .command("push") .description( @@ -1119,6 +1346,7 @@ const command = new Command() "--with-backend-settings ", "Use provided JSON settings instead of querying backend (for testing)", ) + .option("--yes", "Skip interactive prompts and use default behavior") .action(pushGitSyncSettings as any); export { pullGitSyncSettings, pushGitSyncSettings }; diff --git a/cli/main.ts b/cli/main.ts index df61a212e2..0229c65b83 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -143,6 +143,7 @@ const command = new Command() skipFolders: DEFAULT_SYNC_OPTIONS.skipFolders, includeSchedules: DEFAULT_SYNC_OPTIONS.includeSchedules, includeTriggers: DEFAULT_SYNC_OPTIONS.includeTriggers, + overrides: {}, }; await Deno.writeTextFile( diff --git a/cli/sync.ts b/cli/sync.ts index 162c48282e..4f9cb8a715 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -56,43 +56,8 @@ function mergeCliWithEffectiveOptions { assertStringIncludes(wmillYaml, "*.test.ts", "Should include backend's exclude_path"); assertStringIncludes(wmillYaml, "g/**", "Should include backend's extra_include_path"); - // Should NOT have overrides section since we're starting fresh - assertEquals(wmillYaml.includes("overrides:"), false, "Init should not create overrides section"); + // Should have empty overrides section for consistency + assertStringIncludes(wmillYaml, "overrides: {}"); }); }); @@ -180,6 +180,6 @@ Deno.test("Init: --use-default bypasses backend settings check", async () => { // Should NOT have backend-specific settings assertEquals(wmillYaml.includes("f/should-be-ignored/**"), false, "Should not include backend settings"); - assertEquals(wmillYaml.includes("overrides:"), false, "Should not create overrides when using defaults"); + assertStringIncludes(wmillYaml, "overrides: {}", "Should have empty overrides section for consistency"); }); }); \ No newline at end of file diff --git a/cli/test/multi_instance_workspace.test.ts b/cli/test/multi_instance_workspace.test.ts index bfcc86a3b3..97ad4f3ec4 100644 --- a/cli/test/multi_instance_workspace.test.ts +++ b/cli/test/multi_instance_workspace.test.ts @@ -24,7 +24,7 @@ Deno.test("Multi-Instance: gitsync-settings pull with new format", async () => { await withContainerizedBackend(async (backend, tempDir) => { // Set up workspace profile await setupWorkspaceProfile(backend, "multi_instance_test"); - + // Create wmill.yaml with new format overrides for different instances const backendUrl = new URL(backend.baseUrl).toString(); // Normalize URL await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun @@ -38,22 +38,22 @@ overrides: includeTriggers: true includeSchedules: true skipVariables: true - + # Different instance (won't match) "https://app.windmill.dev/:${backend.workspace}:u/test/test_repo": includeTriggers: false includeSchedules: false skipVariables: false`); - + // Pull settings - should use the matching instance override (skipVariables: true) const pullResult = await backend.runCLICommand([ 'gitsync-settings', 'pull', '--repository', 'u/test/test_repo', '--diff' ], tempDir, "multi_instance_test"); - + assertEquals(pullResult.code, 0); - assertStringIncludes(pullResult.stdout, "Changes that would be made:"); + assertStringIncludes(pullResult.stdout, "Changes that would be applied locally:"); // includeSchedules should show as a change since backend default is false assertStringIncludes(pullResult.stdout, "includeSchedules"); }); @@ -61,9 +61,9 @@ overrides: Deno.test("Multi-Instance: gitsync-settings push with overrides", async () => { await withContainerizedBackend(async (backend, tempDir) => { - // Set up workspace profile + // Set up workspace profile await setupWorkspaceProfile(backend, "push_override_test"); - + // Create wmill.yaml with specific settings that differ from backend defaults const backendUrl = new URL(backend.baseUrl).toString(); await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun @@ -78,16 +78,16 @@ overrides: "${backendUrl}:${backend.workspace}:u/test/test_repo": includeSchedules: true skipVariables: true`); - + // Push settings - should show changes because includeSchedules differs from backend const pushResult = await backend.runCLICommand([ 'gitsync-settings', 'push', '--repository', 'u/test/test_repo', '--diff' ], tempDir, "push_override_test"); - + assertEquals(pushResult.code, 0); - assertStringIncludes(pushResult.stdout, "Changes that would be pushed:"); + assertStringIncludes(pushResult.stdout, "Changes that would be pushed to Windmill:"); assertStringIncludes(pushResult.stdout, "includeSchedules"); }); }); @@ -95,7 +95,7 @@ overrides: Deno.test("Multi-Instance: sync with repository-specific overrides", async () => { await withContainerizedBackend(async (backend, tempDir) => { await setupWorkspaceProfile(backend, "my-workspace_123"); - + const backendUrl = new URL(backend.baseUrl).toString(); await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: @@ -105,22 +105,22 @@ excludes: [] overrides: "${backendUrl}:${backend.workspace}:u/test/test_repo": skipApps: true`); - + const result = await backend.runCLICommand([ 'sync', 'pull', '--repository', 'u/test/test_repo', '--dry-run', '--json-output' ], tempDir, "my-workspace_123"); - + assertEquals(result.code, 0); - + const data = parseJsonFromCLIOutput(result.stdout); - + // Test is designed to verify that the new format works correctly - + // The test app should NOT appear in changes because skipApps: true - const hasTestApp = (data.changes || []).some((change: any) => + const hasTestApp = (data.changes || []).some((change: any) => change.path?.includes('f/test_dashboard') ); assertEquals(hasTestApp, false, "Test app should be skipped due to skipApps override"); @@ -130,7 +130,7 @@ overrides: Deno.test("Multi-Instance: auto-detection of single repository override", async () => { await withContainerizedBackend(async (backend, tempDir) => { await setupWorkspaceProfile(backend, "auto_detect_test"); - + const backendUrl = new URL(backend.baseUrl).toString(); await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: @@ -142,21 +142,21 @@ overrides: "${backendUrl}:${backend.workspace}:u/test/test_repo": skipApps: true includeSchedules: true`); - + // Don't specify --repository, it should auto-detect const result = await backend.runCLICommand([ 'sync', 'pull', '--dry-run', '--json-output' ], tempDir, "auto_detect_test"); - + assertEquals(result.code, 0); assertStringIncludes(result.stdout, "Auto-selected repository: u/test/test_repo"); - + const data = parseJsonFromCLIOutput(result.stdout); - + // The test app should NOT appear because of auto-detected skipApps: true - const hasTestApp = (data.changes || []).some((change: any) => + const hasTestApp = (data.changes || []).some((change: any) => change.path?.includes('f/test_dashboard') ); assertEquals(hasTestApp, false, "Test app should be skipped due to auto-detected override"); @@ -166,7 +166,7 @@ overrides: Deno.test("Multi-Instance: workspace wildcards with new format", async () => { await withContainerizedBackend(async (backend, tempDir) => { await setupWorkspaceProfile(backend, "wildcard_test"); - + const backendUrl = new URL(backend.baseUrl).toString(); await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: @@ -178,22 +178,22 @@ overrides: "${backendUrl}:${backend.workspace}:*": skipVariables: true skipResources: true`); - + const result = await backend.runCLICommand([ 'sync', 'pull', '--repository', 'u/test/test_repo', '--dry-run', '--json-output' ], tempDir, "wildcard_test"); - + assertEquals(result.code, 0); - + const data = parseJsonFromCLIOutput(result.stdout); - + // Variables should be skipped due to wildcard override - const hasTestVariable = (data.changes || []).some((change: any) => + const hasTestVariable = (data.changes || []).some((change: any) => change.path?.includes('u/admin/test_config.variable.yaml') ); assertEquals(hasTestVariable, false, "Variables should be skipped due to wildcard override"); }); -}); \ No newline at end of file +}); diff --git a/cli/test/sync_config_resolution.test.ts b/cli/test/sync_config_resolution.test.ts index 7166a54b04..0ec1cc733f 100644 --- a/cli/test/sync_config_resolution.test.ts +++ b/cli/test/sync_config_resolution.test.ts @@ -30,7 +30,7 @@ Deno.test("Integration: wmill.yaml configuration produces expected results", asy await withContainerizedBackend(async (backend, tempDir) => { // Set up workspace profile with name "localhost_test" await setupWorkspaceProfile(backend); - + // Create wmill.yaml with settings await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: @@ -43,7 +43,7 @@ skipResources: true includeSettings: true includeSchedules: true includeTriggers: true`); - + // Test pull with wmill.yaml configuration const yamlResult = await backend.runCLICommand(['sync', 'pull', '--dry-run', '--json-output'], tempDir); if (yamlResult.code !== 0) { @@ -53,21 +53,21 @@ includeTriggers: true`); console.log("Stderr:", yamlResult.stderr); } assertEquals(yamlResult.code, 0); - + // Extract JSON from CLI output (skip log messages) const yamlData = parseJsonFromCLIOutput(yamlResult.stdout); - + // Should include settings.yaml due to includeSettings: true - const hasSettings = (yamlData.changes || []).some((change: any) => + const hasSettings = (yamlData.changes || []).some((change: any) => change.type === 'added' && change.path === 'settings.yaml' ); assertEquals(hasSettings, true); - + // Should NOT include resources or variables (due to skip flags) - const hasResources = (yamlData.changes || []).some((change: any) => + const hasResources = (yamlData.changes || []).some((change: any) => change.type === 'added' && change.path?.includes('.resource.yaml') ); - const hasVariables = (yamlData.changes || []).some((change: any) => + const hasVariables = (yamlData.changes || []).some((change: any) => change.type === 'added' && change.path?.includes('.variable.yaml') ); assertEquals(hasResources, false); @@ -79,35 +79,35 @@ Deno.test("Integration: settings.yaml inclusion respects includeSettings flag", await withContainerizedBackend(async (backend, tempDir) => { // Set up workspace profile with name "localhost_test" await setupWorkspaceProfile(backend); - - // Test 1: includeSettings: true should include settings.yaml + + // Test 1: includeSettings: true should include settings.yaml await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: - "**" includeSettings: true`); - + const includeResult = await backend.runCLICommand(['sync', 'pull', '--dry-run', '--json-output'], tempDir); assertEquals(includeResult.code, 0); - + // Extract JSON from CLI output (skip log messages) const includeData = parseJsonFromCLIOutput(includeResult.stdout); - const hasSettingsInclude = (includeData.changes || []).some((change: any) => + const hasSettingsInclude = (includeData.changes || []).some((change: any) => change.type === 'added' && change.path === 'settings.yaml' ); assertEquals(hasSettingsInclude, true); - + // Test 2: includeSettings: false should NOT include settings.yaml await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: - "**" includeSettings: false`); - + const excludeResult = await backend.runCLICommand(['sync', 'pull', '--dry-run', '--json-output'], tempDir); assertEquals(excludeResult.code, 0); - + // Extract JSON from CLI output (skip log messages) const excludeData = parseJsonFromCLIOutput(excludeResult.stdout); - const hasSettingsExclude = (excludeData.changes || []).some((change: any) => + const hasSettingsExclude = (excludeData.changes || []).some((change: any) => change.type === 'added' && change.path === 'settings.yaml' ); assertEquals(hasSettingsExclude, false); @@ -118,30 +118,86 @@ Deno.test("Integration: resource/variable filtering respects skip flags", async await withContainerizedBackend(async (backend, tempDir) => { // Set up workspace profile with name "localhost_test" await setupWorkspaceProfile(backend); - - // Test skipResources: true + + // Test skipResources: true await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: - "**" skipResources: true skipVariables: false`); - + const result = await backend.runCLICommand(['sync', 'pull', '--dry-run', '--json-output'], tempDir); assertEquals(result.code, 0); - + // Extract JSON from CLI output (skip log messages) const data = parseJsonFromCLIOutput(result.stdout); - + // Should NOT include resources - const hasResources = (data.changes || []).some((change: any) => + const hasResources = (data.changes || []).some((change: any) => change.type === 'added' && change.path?.includes('.resource.yaml') ); assertEquals(hasResources, false); - + // Should include variables (not skipped) - const hasVariables = (data.changes || []).some((change: any) => + const hasVariables = (data.changes || []).some((change: any) => change.type === 'added' && change.path?.includes('.variable.yaml') ); assertEquals(hasVariables, true); }); -}); \ No newline at end of file +}); + +// ============================================================================= +// CLI FLAG OVERRIDE TESTS +// Tests for CLI flags overriding configuration file settings +// ============================================================================= + +Deno.test("CLI skip flags override wmill.yaml configuration", async () => { + await withContainerizedBackend(async (backend, tempDir) => { + // Set up workspace profile with name "localhost_test" + await setupWorkspaceProfile(backend); + + // Create wmill.yaml that INCLUDES resources by default (skipResources: false) + await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: + - f/** + - u/** +skipResources: false +skipResourceTypes: false +includeSettings: true`); + + // Test 1: Without CLI flags - should respect wmill.yaml (include resources) + const configResult = await backend.runCLICommand(['sync', 'pull', '--dry-run', '--json-output'], tempDir); + assertEquals(configResult.code, 0); + + const configData = parseJsonFromCLIOutput(configResult.stdout); + + + // Should include resources (not skipped by config) + const hasResources = (configData.changes || []).some((change: any) => + change.type === 'added' && change.path?.includes('.resource.yaml') + ); + assertEquals(hasResources, true, "Resources should be included by wmill.yaml config"); + + // Test 2: With CLI --skip-resources flag - should override wmill.yaml to skip resources + const overrideResult = await backend.runCLICommand([ + 'sync', 'pull', '--dry-run', '--json-output', + '--skip-resources', // CLI flag should override config to skip resources + '--skip-resource-types' // CLI flag should override config to skip resource types + ], tempDir); + assertEquals(overrideResult.code, 0); + + const overrideData = parseJsonFromCLIOutput(overrideResult.stdout); + + // Should NOT include resources (CLI flag overrides config) + const hasResourcesOverride = (overrideData.changes || []).some((change: any) => + change.type === 'added' && change.path?.includes('.resource.yaml') + ); + assertEquals(hasResourcesOverride, false, "CLI --skip-resources flag should override wmill.yaml to exclude resources"); + + // Should NOT include resource types (CLI flag overrides config) + const hasResourceTypesOverride = (overrideData.changes || []).some((change: any) => + change.type === 'added' && change.path?.includes('.resource-type.yaml') + ); + assertEquals(hasResourceTypesOverride, false, "CLI --skip-resource-types flag should override wmill.yaml to exclude resource types"); + }); +}); diff --git a/cli/utils.ts b/cli/utils.ts index 561cb6504b..af5548330c 100644 --- a/cli/utils.ts +++ b/cli/utils.ts @@ -2,7 +2,7 @@ // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-nocheck This file is copied from a JS project, so it's not type-safe. -import { log, encodeHex, SEP } from "./deps.ts"; +import { colors, log, encodeHex, SEP } from "./deps.ts"; import crypto from "node:crypto"; export function deepEqual(a: T, b: T): boolean { @@ -167,7 +167,7 @@ export async function selectRepository( if (repositories.length === 1) { const repoPath = repositories[0].git_repo_resource_path.replace(/^\$res:/, ""); - log.info(`Using repository: ${repoPath}`); + log.info(colors.cyan(`Auto-selected repository: ${colors.bold(repoPath)}`)); return repositories[0]; } From 2ab5345e617cbf336447768232c6f6e39cd406a3 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Fri, 18 Jul 2025 00:15:01 +0200 Subject: [PATCH 22/22] Assets refactor (#6217) * Moved logic to FlowAssetsProvider * Remove assetsMap in flow * do not parse everything on mount + only check for missing assets fields * add assets field in backend * remove fallbackAccessTypes * better structure and less queries / parsing * Fix assets not showing when pulling raw_flow from jobs * flow assets ctx for job run * Fix transitive assets fetching * Fix input args asset node * enablePathScriptAndFlowAssets flag * edit btn for variable * untrack refresh * move parseInputArgsAssets * Assets tab in runs * Update FlowStatusViewerInner to svelte 5 + fix asset sync bug * avoid toast error on bad resource * fetch res metadata for input arg asset * Job assets viewer in run page * r/w selector * remove indigo badge * store alt_access_type state in ScriptEditor * Don't parse assets in flow script editor * Add alt_access_type in backend * show Read as selected by default to avoid giving the feeling of having made a decision * keep alt_access_type when reparsing in flow raw scripts * Remove variable asset kind, and save assets for scripts * remove all backend asset parsing * R/W/RW selector button nits * fix insert into assets not saving alt access type * support named arguments in python asset parser * improve asset usage drawer R/W indicator * update legacy $res: syntax * reactivity issue * remove last variable asset stuff * sqlx prepare * tooltip explainer * deprecated variable asset nit --- ...b21270131e9e93ca10d195664e7e5a774fe9e.json | 91 +++++++ .../20250717135503_assets_refactor.down.sql | 7 + .../20250717135503_assets_refactor.up.sql | 8 + .../windmill-parser-py/src/asset_parser.rs | 36 +-- .../windmill-parser-ts/src/asset_parser.rs | 2 - .../windmill-parser/src/asset_parser.rs | 3 - backend/tests/worker.rs | 12 +- backend/windmill-api/openapi.yaml | 16 +- backend/windmill-api/src/flows.rs | 2 +- backend/windmill-api/src/scripts.rs | 24 +- backend/windmill-common/src/assets.rs | 79 +----- backend/windmill-common/src/flows.rs | 14 +- backend/windmill-common/src/scripts.rs | 8 +- .../windmill-worker/src/worker_lockfiles.rs | 13 +- frontend/src/lib/components/EditorBar.svelte | 4 +- .../src/lib/components/FlowBuilder.svelte | 13 + .../lib/components/FlowStatusViewer.svelte | 4 +- .../components/FlowStatusViewerInner.svelte | 243 +++++++++++------- .../src/lib/components/ScriptBuilder.svelte | 7 +- .../src/lib/components/ScriptEditor.svelte | 50 ++-- .../lib/components/assets/AssetButtons.svelte | 56 ++++ .../assets/AssetsDropdownButton.svelte | 141 +++++----- .../assets/AssetsUsageDrawer.svelte | 21 +- .../components/assets/JobAssetsViewer.svelte | 95 +++++++ .../src/lib/components/assets/README_DEV.md | 0 frontend/src/lib/components/assets/lib.ts | 63 ++++- .../components/flows/FlowAssetsHandler.svelte | 162 ++++++++++++ .../flows/content/FlowModuleComponent.svelte | 25 +- frontend/src/lib/components/flows/types.ts | 3 +- .../lib/components/graph/FlowGraphV2.svelte | 128 +-------- .../components/graph/graphBuilder.svelte.ts | 31 ++- .../graph/renderers/edges/BaseEdge.svelte | 15 +- .../graph/renderers/nodes/AssetNode.svelte | 71 ++--- .../nodes/AssetsOverflowedNode.svelte | 4 +- .../components/icons/AssetGenericIcon.svelte | 3 - .../lib/components/icons/AssetVarIcon.svelte | 38 --- .../src/lib/components/runs/JobPreview.svelte | 17 +- frontend/src/lib/components/script_builder.ts | 4 +- frontend/src/lib/svelte5Utils.svelte.ts | 40 +-- frontend/src/lib/utils.ts | 17 ++ .../(logged)/flows/get/[...path]/+page.svelte | 14 + .../(root)/(logged)/run/[...run]/+page.svelte | 25 +- openflow.openapi.yaml | 10 +- 43 files changed, 994 insertions(+), 625 deletions(-) create mode 100644 backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json create mode 100644 backend/migrations/20250717135503_assets_refactor.down.sql create mode 100644 backend/migrations/20250717135503_assets_refactor.up.sql create mode 100644 frontend/src/lib/components/assets/AssetButtons.svelte create mode 100644 frontend/src/lib/components/assets/JobAssetsViewer.svelte create mode 100644 frontend/src/lib/components/assets/README_DEV.md create mode 100644 frontend/src/lib/components/flows/FlowAssetsHandler.svelte delete mode 100644 frontend/src/lib/components/icons/AssetVarIcon.svelte diff --git a/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json b/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json new file mode 100644 index 0000000000..e8b39caef7 --- /dev/null +++ b/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json @@ -0,0 +1,91 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Varchar", + "Int8Array", + "Text", + "Text", + "Text", + "Varchar", + "Text", + "Bool", + "Jsonb", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb" + ] + } + } + }, + { + "Custom": { + "name": "script_kind", + "kind": { + "Enum": [ + "script", + "trigger", + "failure", + "command", + "approval", + "preprocessor" + ] + } + } + }, + "Varchar", + "Bool", + "VarcharArray", + "Int4", + "Int4", + "Int4", + "Bool", + "Bool", + "Int2", + "Bool", + "Bool", + "Int4", + "Varchar", + "Bool", + "Bool", + "Varchar", + "Bool", + "Text", + "Bool", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e" +} diff --git a/backend/migrations/20250717135503_assets_refactor.down.sql b/backend/migrations/20250717135503_assets_refactor.down.sql new file mode 100644 index 0000000000..76780e9200 --- /dev/null +++ b/backend/migrations/20250717135503_assets_refactor.down.sql @@ -0,0 +1,7 @@ +-- Add 'variable' kind back +ALTER TABLE asset ALTER column kind TYPE VARCHAR; +DROP TYPE asset_kind; +CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource', 'variable'); +ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND; + +ALTER TABLE script DROP COLUMN assets; diff --git a/backend/migrations/20250717135503_assets_refactor.up.sql b/backend/migrations/20250717135503_assets_refactor.up.sql new file mode 100644 index 0000000000..bd17555a9c --- /dev/null +++ b/backend/migrations/20250717135503_assets_refactor.up.sql @@ -0,0 +1,8 @@ +ALTER TABLE script ADD COLUMN assets jsonb; + +-- Remove 'variable' kind +DELETE FROM asset WHERE kind = 'variable'; +ALTER TABLE asset ALTER column kind TYPE VARCHAR; +DROP TYPE asset_kind; +CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource'); +ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND; \ No newline at end of file diff --git a/backend/parsers/windmill-parser-py/src/asset_parser.rs b/backend/parsers/windmill-parser-py/src/asset_parser.rs index d6f16824c7..a96057af1e 100644 --- a/backend/parsers/windmill-parser-py/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-py/src/asset_parser.rs @@ -57,27 +57,27 @@ impl AssetsFinder { }) .ok_or(())?; + use AssetKind::*; let (kind, access_type, arg) = match ident.as_str() { - "load_s3_file" => (AssetKind::S3Object, Some(R), Arg::Pos(0)), - "load_s3_file_reader" => (AssetKind::S3Object, Some(R), Arg::Pos(0)), - "write_s3_file" => (AssetKind::S3Object, Some(W), Arg::Pos(0)), - "get_resource" => (AssetKind::Resource, None, Arg::Pos(0)), - "set_resource" => (AssetKind::Resource, Some(W), Arg::Named("path")), - "get_boto3_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), - "get_polars_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), - "get_duckdb_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), - "get_variable" => (AssetKind::Variable, Some(R), Arg::Pos(0)), - "set_variable" => (AssetKind::Variable, Some(W), Arg::Pos(0)), + "load_s3_file" => (S3Object, Some(R), Arg::Pos(0, "s3object")), + "load_s3_file_reader" => (S3Object, Some(R), Arg::Pos(0, "s3object")), + "write_s3_file" => (S3Object, Some(W), Arg::Pos(0, "s3object")), + "get_resource" => (Resource, None, Arg::Pos(0, "path")), + "set_resource" => (Resource, Some(W), Arg::Pos(0, "path")), + "get_boto3_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), + "get_polars_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), + "get_duckdb_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), _ => return Err(()), }; let arg_val = match arg { - Arg::Pos(i) => node.args.get(i), - Arg::Named(name) => node - .keywords - .iter() - .find(|kw| kw.arg.as_deref() == Some(name)) - .map(|kw| &kw.value), + Arg::Pos(i, name) => node.args.get(i).or_else(|| { + // Get arg by name + node.keywords + .iter() + .find(|kw| kw.arg.as_deref() == Some(name)) + .map(|kw| &kw.value) + }), }; match arg_val { @@ -93,6 +93,6 @@ impl AssetsFinder { } enum Arg { - Pos(usize), - Named(&'static str), + // Positional arguments in python can also be used by their name + Pos(usize, &'static str), } diff --git a/backend/parsers/windmill-parser-ts/src/asset_parser.rs b/backend/parsers/windmill-parser-ts/src/asset_parser.rs index dcd6681f5f..6b81272ec1 100644 --- a/backend/parsers/windmill-parser-ts/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-ts/src/asset_parser.rs @@ -84,8 +84,6 @@ impl AssetsFinder { "denoS3LightClientSettings" => (AssetKind::Resource, None, 0), "duckdbConnectionSettings" => (AssetKind::Resource, None, 0), "polarsConnectionSettings" => (AssetKind::Resource, None, 0), - "getVariable" => (AssetKind::Variable, Some(R), 0), - "setVariable" => (AssetKind::Variable, Some(W), 0), _ => return Err(()), }; diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 893843342b..6857e8d68e 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -15,7 +15,6 @@ use AssetUsageAccessType::*; pub enum AssetKind { S3Object, Resource, - Variable, } #[derive(Serialize)] @@ -57,8 +56,6 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> { Some((AssetKind::Resource, &s[6..])) } else if s.starts_with("$res:") { Some((AssetKind::Resource, &s[5..])) - } else if s.starts_with("var://") { - Some((AssetKind::Variable, &s[6..])) } else { None } diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 28bce55024..65263152dc 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1147,7 +1147,7 @@ async fn test_deno_flow(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, } .into(), stop_after_if: Default::default(), @@ -1191,7 +1191,7 @@ async fn test_deno_flow(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, } .into(), stop_after_if: Default::default(), @@ -1321,7 +1321,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), @@ -1376,7 +1376,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), stop_after_all_iters_if: Default::default(), @@ -1416,7 +1416,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), @@ -1482,7 +1482,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), stop_after_all_iters_if: Default::default(), diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 777c708c93..e2259a4900 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -3115,7 +3115,7 @@ paths: type: array items: $ref: "#/components/schemas/ContextualVariable" - + /w/{workspace}/workspaces/get_secondary_storage_names: get: summary: get secondary storage names @@ -13838,20 +13838,27 @@ components: type: boolean on_behalf_of_email: type: string - fallback_access_types: + assets: type: array items: type: object - required: [path, kind, access_type] + required: + - path + - kind properties: path: type: string kind: type: string - enum: [s3object, resource] + enum: + - s3object + - resource access_type: type: string enum: [r, w, rw] + alt_access_type: + type: string + enum: [r, w, rw] required: - path @@ -17606,7 +17613,6 @@ components: enum: - s3object - resource - - variable Asset: type: object properties: diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 4ebeecb675..f9d7827341 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -1391,7 +1391,7 @@ mod tests { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }), stop_after_if: Some(StopAfterIf { expr: "foo = 'bar'".to_string(), diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 98c65c1656..e32a2da82b 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -43,7 +43,7 @@ use windmill_audit::ActionKind; use windmill_worker::process_relative_imports; use windmill_common::{ - assets::{clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind}, + assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType}, error::to_anyhow, worker::CLOUD_HOSTED, }; @@ -116,6 +116,9 @@ pub struct ScriptWDraft { pub has_preprocessor: Option, #[serde(skip_serializing_if = "Option::is_none")] pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[sqlx(json(nullable))] + pub assets: Option>, } pub fn global_service() -> Router { @@ -771,8 +774,8 @@ async fn create_script_internal<'c>( content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33)", + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)", &w_id, &hash.0, ns.path, @@ -810,6 +813,7 @@ async fn create_script_internal<'c>( None }, validate_schema, + ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok()) ) .execute(&mut *tx) .await?; @@ -924,16 +928,8 @@ async fn create_script_internal<'c>( } clear_asset_usage(&mut *tx, &w_id, &script_path, AssetUsageKind::Script).await?; - for asset in parse_assets(&ns.content, ns.language)?.iter().flatten() { - insert_asset_usage( - &mut *tx, - &w_id, - asset, - ns.fallback_access_types.as_ref().map(Vec::as_slice), - &ns.path, - AssetUsageKind::Script, - ) - .await?; + for asset in ns.assets.as_ref().into_iter().flatten() { + insert_asset_usage(&mut *tx, &w_id, &asset, &ns.path, AssetUsageKind::Script).await?; } let permissioned_as = username_to_permissioned_as(&authed.username); @@ -1126,7 +1122,7 @@ async fn get_script_by_path_w_draft( let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 ORDER BY script.created_at DESC LIMIT 1", diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 6b720b368d..20c711c127 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -1,8 +1,7 @@ use serde::{Deserialize, Serialize}; use sqlx::PgExecutor; -use windmill_parser::asset_parser::ParseAssetsResult; -use crate::{error, scripts::ScriptLang}; +use crate::error; #[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] #[sqlx(type_name = "ASSET_KIND", rename_all = "lowercase")] @@ -10,7 +9,8 @@ use crate::{error, scripts::ScriptLang}; pub enum AssetKind { S3Object, Resource, - Variable, + // Avoid unnexpected crashes when deserializing old assets + Variable, // Deprecated } #[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] @@ -41,85 +41,28 @@ pub struct AssetUsage { pub access_type: AssetUsageAccessType, } -#[derive(Serialize, Deserialize, Debug, Clone, Hash)] -pub struct AssetWithAccessType { +#[derive(Serialize, Deserialize, Debug, Clone, Hash, sqlx::Type)] +pub struct AssetWithAltAccessType { pub path: String, pub kind: AssetKind, - pub access_type: AssetUsageAccessType, -} - -pub fn parse_assets( - input: &str, - lang: ScriptLang, -) -> anyhow::Result>>> { - let r = match lang { - ScriptLang::Python3 => windmill_parser_py::parse_assets(input), - ScriptLang::DuckDb => windmill_parser_sql::parse_assets(input).map(|a| { - a.iter() - .map(|a| ParseAssetsResult { - path: a.path.to_string(), - access_type: a.access_type, - kind: a.kind, - }) - .collect() - }), - ScriptLang::Deno | ScriptLang::Bun | ScriptLang::Nativets => { - windmill_parser_ts::parse_assets(input) - } - _ => return Ok(None), - }; - return r.map(Some); -} - -impl From for AssetKind { - fn from(kind: windmill_parser::asset_parser::AssetKind) -> Self { - match kind { - windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object, - windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, - windmill_parser::asset_parser::AssetKind::Variable => AssetKind::Variable, - } - } -} -impl From for AssetUsageAccessType { - fn from(access_type: windmill_parser::asset_parser::AssetUsageAccessType) -> Self { - match access_type { - windmill_parser::asset_parser::AssetUsageAccessType::R => AssetUsageAccessType::R, - windmill_parser::asset_parser::AssetUsageAccessType::W => AssetUsageAccessType::W, - windmill_parser::asset_parser::AssetUsageAccessType::RW => AssetUsageAccessType::RW, - } - } + pub access_type: Option, + pub alt_access_type: Option, } pub async fn insert_asset_usage<'e>( executor: impl PgExecutor<'e>, workspace_id: &str, - parsed_asset: &ParseAssetsResult, - fallback_access_types: Option<&[AssetWithAccessType]>, + asset: &AssetWithAltAccessType, usage_path: &str, usage_kind: AssetUsageKind, ) -> error::Result<()> { - let kind: AssetKind = parsed_asset.kind.into(); - let asset_alternative_access_type = || { - fallback_access_types - .as_ref() - .and_then(|v| { - v.iter() - .find(|a| a.kind == kind && a.path == parsed_asset.path) - }) - .map(|a| a.access_type) - }; - let access_type: Option = parsed_asset - .access_type - .map(Into::into) - .or_else(asset_alternative_access_type); - sqlx::query!( r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING"#, workspace_id, - parsed_asset.path, - kind as AssetKind, - access_type as Option, + asset.path, + asset.kind as AssetKind, + (asset.access_type.or(asset.alt_access_type)) as Option, usage_path, usage_kind as AssetUsageKind ) diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index b6d8f422e2..4a4cb200a8 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -18,7 +18,7 @@ use sqlx::types::Json; use sqlx::types::JsonRawValue; use crate::{ - assets::AssetWithAccessType, + assets::AssetWithAltAccessType, cache, error::Error, more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, @@ -506,7 +506,7 @@ pub enum FlowModuleValue { #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, #[serde(skip_serializing_if = "Option::is_none")] - asset_fallback_access_types: Option>, + assets: Option>, }, Identity, // Internal only, never exposed to the frontend. @@ -526,6 +526,8 @@ pub enum FlowModuleValue { concurrency_time_window_s: Option, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, + #[serde(skip_serializing_if = "Option::is_none")] + assets: Option>, }, } @@ -560,7 +562,7 @@ struct UntaggedFlowModuleValue { id: Option, default_node: Option, modules_node: Option, - asset_fallback_access_types: Option>, + assets: Option>, } impl<'de> Deserialize<'de> for FlowModuleValue { @@ -635,7 +637,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue { concurrent_limit: untagged.concurrent_limit, concurrency_time_window_s: untagged.concurrency_time_window_s, is_trigger: untagged.is_trigger, - asset_fallback_access_types: untagged.asset_fallback_access_types, + assets: untagged.assets, }), "flowscript" => Ok(FlowModuleValue::FlowScript { input_transforms: untagged.input_transforms.unwrap_or_default(), @@ -650,6 +652,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue { concurrent_limit: untagged.concurrent_limit, concurrency_time_window_s: untagged.concurrency_time_window_s, is_trigger: untagged.is_trigger, + assets: untagged.assets, }), "identity" => Ok(FlowModuleValue::Identity), other => Err(serde::de::Error::unknown_variant( @@ -785,6 +788,7 @@ pub async fn resolve_module( concurrent_limit, concurrency_time_window_s, is_trigger, + assets, } = std::mem::replace(&mut val, Identity) else { unreachable!() @@ -808,7 +812,7 @@ pub async fn resolve_module( concurrent_limit, concurrency_time_window_s, is_trigger, - asset_fallback_access_types: None, + assets, }; } ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => { diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 57dd7d2498..ff2f1865af 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -13,7 +13,7 @@ use std::{ }; use crate::{ - assets::AssetWithAccessType, + assets::AssetWithAltAccessType, error::{to_anyhow, Error}, utils::http_get_from_hub, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, @@ -250,6 +250,9 @@ pub struct Script { pub has_preprocessor: Option, #[serde(skip_serializing_if = "Option::is_none")] pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[sqlx(json(nullable))] + pub assets: Option>, } #[derive(Serialize, sqlx::FromRow)] @@ -349,7 +352,8 @@ pub struct NewScript { pub codebase: Option, pub has_preprocessor: Option, pub on_behalf_of_email: Option, - pub fallback_access_types: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub assets: Option>, } fn lock_deserialize<'de, D>(deserializer: D) -> Result, D::Error> diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index ffd6909032..18720456c5 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -11,9 +11,7 @@ use serde_json::{json, Value}; use sha2::Digest; use sqlx::types::Json; use uuid::Uuid; -use windmill_common::assets::{ - clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind, -}; +use windmill_common::assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind}; use windmill_common::error::Error; use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; @@ -946,7 +944,7 @@ async fn lock_modules<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, - asset_fallback_access_types, + assets, } = e.get_value()? else { match e.get_value()? { @@ -1139,12 +1137,11 @@ async fn lock_modules<'c>( continue; }; - for asset in parse_assets(&content, language)?.iter().flatten() { + for asset in assets.iter().flatten() { insert_asset_usage( &mut *tx, &job.workspace_id, asset, - asset_fallback_access_types.as_ref().map(Vec::as_slice), job_path, AssetUsageKind::Flow, ) @@ -1266,7 +1263,7 @@ async fn lock_modules<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, - asset_fallback_access_types, + assets, }); new_flow_modules.push(e); @@ -1418,6 +1415,7 @@ async fn reduce_flow<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, + assets, .. } = std::mem::replace(&mut val, Identity) else { @@ -1436,6 +1434,7 @@ async fn reduce_flow<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, + assets, }; } ForloopFlow { modules, modules_node, .. } diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 41dc59c0f2..749f8d163d 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -593,10 +593,10 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS let t = { postgresql: 'postgres', mysql: 'mysql', bigquery: 'bigquery' }[resType] if (!t) { sendUserToast(`Resource type ${resType} is not supported in DuckDB`, true) - editor.insertAtCursor(`'$res:${path}'`) + editor.insertAtCursor(`'res://${path}'`) return } else { - editor.insertAtCursor(`ATTACH '$res:${path}' AS db (TYPE ${t});`) + editor.insertAtCursor(`ATTACH 'res://${path}' AS db (TYPE ${t});`) } } diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 2672b491de..da9fb506ca 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -86,6 +86,7 @@ } from './stepHistoryLoader.svelte' import type { FlowBuilderProps } from './flow_builder' import { ModulesTestStates } from './modulesTest.svelte' + import FlowAssetsHandler, { initFlowGraphAssetsCtx } from './flows/FlowAssetsHandler.svelte' let { initialPath = $bindable(''), @@ -609,6 +610,11 @@ outputPickerOpenFns }) + setContext( + 'FlowGraphAssetContext', + initFlowGraphAssetsCtx({ getModules: () => flowStore.val.value.modules }) + ) + // Add triggers context store const triggersState = $state( new Triggers( @@ -1256,3 +1262,10 @@ renderCount += 1 }} /> + + diff --git a/frontend/src/lib/components/FlowStatusViewer.svelte b/frontend/src/lib/components/FlowStatusViewer.svelte index 9afcd99f85..305f9d3c20 100644 --- a/frontend/src/lib/components/FlowStatusViewer.svelte +++ b/frontend/src/lib/components/FlowStatusViewer.svelte @@ -74,8 +74,8 @@ }} globalModuleStates={[]} globalDurationStatuses={[]} - bind:localModuleStates - bind:localDurationStatuses + {localModuleStates} + {localDurationStatuses} bind:selectedNode={selectedJobStep} on:start on:done diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index dd8f70eb17..0060fc1ade 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -1,4 +1,6 @@ {#if notAnonynmous} @@ -906,7 +950,7 @@

For performance reasons, only the last 20 items are shown by default

diff --git a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte index 6c2722ffae..f451b36bb9 100644 --- a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte +++ b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte @@ -1,20 +1,26 @@ + +
    + {#each assets.value ?? [] as asset} +
  • +
    + {asset.path} + + {formatAssetKind({ + ...asset, + ...(asset.kind === 'resource' + ? { metadata: { resource_type: resourceDataCache[asset.path] } } + : {}) + })} + +
    + +
    • + {/each} +
+ + + + diff --git a/frontend/src/lib/components/assets/README_DEV.md b/frontend/src/lib/components/assets/README_DEV.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index a30e48419d..a58f94de4f 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -2,13 +2,17 @@ import type { AssetKind as _AssetKind, Asset as _Asset, ListAssetsResponse, - AssetUsageAccessType + AssetUsageAccessType, + FlowModule, + ScriptArgs } from '$lib/gen' import { capitalize } from '$lib/utils' - export type Asset = _Asset export type AssetKind = _AssetKind export type AssetWithAccessType = Asset & { access_type?: AssetUsageAccessType } +export type AssetWithAltAccessType = AssetWithAccessType & { + alt_access_type?: AssetUsageAccessType +} export function formatAsset(asset: Asset): string { switch (asset.kind) { @@ -16,8 +20,6 @@ export function formatAsset(asset: Asset): string { return `res://${asset.path}` case 's3object': return `s3://${asset.path}` - case 'variable': - return `var://${asset.path}` } } @@ -34,6 +36,11 @@ export function assetEq(a: Asset | undefined, b: Asset | undefined): boolean { return a.kind === b.kind && a.path === b.path } +export function assetsEq(a: Asset[], b: Asset[]): boolean { + if (a.length !== b.length) return false + return a.every((asset, index) => assetEq(asset, b[index])) +} + export function parseAssetFromString(s: string): Asset | undefined { if (s.startsWith('res://')) { return { kind: 'resource', path: s.slice(6) } @@ -41,8 +48,6 @@ export function parseAssetFromString(s: string): Asset | undefined { return { kind: 'resource', path: s.slice(5) } } else if (s.startsWith('s3://')) { return { kind: 's3object', path: s.slice(5) } - } else if (s.startsWith('var://')) { - return { kind: 'variable', path: s.slice(6) } } return undefined } @@ -63,7 +68,49 @@ export function formatAssetKind(asset: { } case 's3object': return 'S3 Object' - case 'variable': - return 'Variable' } } + +export function formatAssetAccessType(accessType: AssetUsageAccessType | undefined) { + switch (accessType) { + case 'r': + return 'Read' + case 'w': + return 'Write' + case 'rw': + return 'R/W' + } + return '?' +} + +export function getAccessType(asset: AssetWithAltAccessType): AssetUsageAccessType | undefined { + if (asset.alt_access_type) return asset.alt_access_type + if (asset.access_type) return asset.access_type +} + +export function getFlowModuleAssets( + flowModuleValue: FlowModule, + additionalAssetsMap?: Record +): AssetWithAccessType[] | undefined { + if (flowModuleValue.value.type === 'rawscript') return flowModuleValue.value.assets + if (flowModuleValue.value.type === 'script' || flowModuleValue.value.type === 'flow') { + const additionalAssets = additionalAssetsMap?.[flowModuleValue.id] + if (additionalAssets) return additionalAssets + } + return undefined +} + +export function parseInputArgsAssets(args: ScriptArgs): AssetWithAccessType[] { + const arr: AssetWithAccessType[] = [] + for (const v of Object.values(args)) { + if (typeof v === 'string') { + const asset = parseAssetFromString(v) + if (asset) arr.push(asset) + } else if (v && typeof v === 'object' && typeof v['s3'] === 'string') { + const s3 = v['s3'] + const storage = typeof v['storage'] == 'string' ? v['storage'] : undefined + arr.push({ kind: 's3object', path: `${storage ?? ''}/${s3}` }) + } + } + return arr +} diff --git a/frontend/src/lib/components/flows/FlowAssetsHandler.svelte b/frontend/src/lib/components/flows/FlowAssetsHandler.svelte new file mode 100644 index 0000000000..cd4ebc98c4 --- /dev/null +++ b/frontend/src/lib/components/flows/FlowAssetsHandler.svelte @@ -0,0 +1,162 @@ + + + + +{#if enableParser} + {#each allModules as mod (mod.id)} + {#if mod.value.type === 'rawscript'} + {@const v = mod.value} + parseAndUpdateRawScriptModule(v)} /> + {/if} + {/each} +{/if} + +{#if flowGraphAssetsCtx} + + + +{/if} diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index 58dd9ae29a..48fec0db83 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -51,10 +51,9 @@ import { workspaceStore } from '$lib/stores' import { checkIfParentLoop } from '../utils' import ModulePreviewResultViewer from '$lib/components/ModulePreviewResultViewer.svelte' - import { refreshStateStore, usePromise } from '$lib/svelte5Utils.svelte' + import { refreshStateStore } from '$lib/svelte5Utils.svelte' import { getStepHistoryLoaderContext } from '$lib/components/stepHistoryLoader.svelte' import AssetsDropdownButton from '$lib/components/assets/AssetsDropdownButton.svelte' - import { inferAssets } from '$lib/infer' const { selectedId, @@ -123,6 +122,8 @@ let testIsLoading = $state(false) let scriptProgress = $state(undefined) + let assets = $derived((flowModule.value.type === 'rawscript' && flowModule.value.assets) || []) + function onModulesChange(savedModule: FlowModule | undefined, flowModule: FlowModule) { // console.log('onModulesChange', savedModule, flowModule) return savedModule?.value?.type === 'rawscript' && @@ -299,19 +300,6 @@ } }) - let assets = usePromise( - async () => - flowModule.value.type === 'rawscript' - ? await inferAssets(flowModule.value.language, flowModule.value.content) - : undefined, - { clearValueOnRefresh: false, loadInit: false } - ) - $effect(() => { - if (flowModule.value.type !== 'rawscript') return - ;[flowModule.value.content, flowModule.value.language] - untrack(() => assets.refresh()) - }) - let rawScriptLang = $derived( flowModule.value.type == 'rawscript' ? flowModule.value.language : undefined ) @@ -422,11 +410,8 @@ {#if !noEditor} {#key flowModule.id}
- {#if assets.value?.length} - + {#if assets?.length} + {/if}
// Maps module ids to their assets s3FilePicker: S3FilePicker | undefined dbManagerDrawer: DbManagerDrawer | undefined resourceEditorDrawer: ResourceEditorDrawer | undefined // Maps resource paths to their metadata. undefined is for error resourceMetadataCache: Record + additionalAssetsMap: Record + computeAssetsCount: (asset: Asset) => number }> diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index ef4bcef38b..bfdeca1e21 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -1,12 +1,5 @@ @@ -249,8 +245,10 @@ 'bg-surface h-6 flex items-center gap-1.5 rounded-sm text-tertiary border overflow-clip', isSelected ? 'bg-surface-secondary !border-surface-inverse' : 'border-transparent' )} - onmouseenter={() => (flowGraphAssetsCtx.val.selectedAsset = data.asset)} - onmouseleave={() => (flowGraphAssetsCtx.val.selectedAsset = undefined)} + onmouseenter={() => + flowGraphAssetsCtx && (flowGraphAssetsCtx.val.selectedAsset = data.asset)} + onmouseleave={() => + flowGraphAssetsCtx && (flowGraphAssetsCtx.val.selectedAsset = undefined)} > {/if}
- Used in {pluralize(usageCount, 'step')}
+ Used in {pluralize( + flowGraphAssetsCtx?.val.computeAssetsCount?.(data.asset) ?? -1, + 'step' + )}
{ if (data.asset.kind === 'resource') - flowGraphAssetsCtx.val.resourceEditorDrawer?.initEdit(data.asset.path) + flowGraphAssetsCtx?.val.resourceEditorDrawer?.initEdit(data.asset.path) }} > {data.asset.path} @@ -298,6 +299,8 @@ {formatAssetKind({ ...data.asset, metadata: cachedResourceMetadata })} +
+ fdsfs {/snippet} diff --git a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte index 994983956d..88fb4b1059 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte @@ -14,12 +14,12 @@ data: AssetsOverflowedN['data'] } let { data }: Props = $props() - const flowGraphAssetsCtx = getContext('FlowGraphAssetContext') + const flowGraphAssetsCtx = getContext('FlowGraphAssetContext') let isOpen = $state(false) let includesSelected = $derived( - data.overflowedAssets.some((asset) => assetEq(flowGraphAssetsCtx.val.selectedAsset, asset)) + data.overflowedAssets.some((asset) => assetEq(flowGraphAssetsCtx?.val.selectedAsset, asset)) ) let wasOpenedBecauseOfExternalSelected = false diff --git a/frontend/src/lib/components/icons/AssetGenericIcon.svelte b/frontend/src/lib/components/icons/AssetGenericIcon.svelte index 5be0c0c438..be8f4feb58 100644 --- a/frontend/src/lib/components/icons/AssetGenericIcon.svelte +++ b/frontend/src/lib/components/icons/AssetGenericIcon.svelte @@ -3,7 +3,6 @@ import type { AssetKind } from '../assets/lib' import AssetResIcon from './AssetResIcon.svelte' import AssetS3Icon from './AssetS3Icon.svelte' - import AssetVarIcon from './AssetVarIcon.svelte' interface Props { size?: string @@ -19,8 +18,6 @@ {:else if assetKind == 'resource'} -{:else if assetKind == 'variable'} - {:else} {/if} diff --git a/frontend/src/lib/components/icons/AssetVarIcon.svelte b/frontend/src/lib/components/icons/AssetVarIcon.svelte deleted file mode 100644 index be24443034..0000000000 --- a/frontend/src/lib/components/icons/AssetVarIcon.svelte +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - diff --git a/frontend/src/lib/components/runs/JobPreview.svelte b/frontend/src/lib/components/runs/JobPreview.svelte index 47640bfcae..e99fe4414e 100644 --- a/frontend/src/lib/components/runs/JobPreview.svelte +++ b/frontend/src/lib/components/runs/JobPreview.svelte @@ -15,8 +15,10 @@ import WorkflowTimeline from '../WorkflowTimeline.svelte' import Popover from '../Popover.svelte' import { isFlowPreview, isScriptPreview, truncateRev } from '$lib/utils' - import { createEventDispatcher, untrack } from 'svelte' + import { createEventDispatcher, setContext, untrack } from 'svelte' import { ListFilter } from 'lucide-svelte' + import FlowAssetsHandler, { initFlowGraphAssetsCtx } from '../flows/FlowAssetsHandler.svelte' + import JobAssetsViewer from '../assets/JobAssetsViewer.svelte' interface Props { id: string @@ -46,6 +48,11 @@ let viewTab = $state('result') + setContext( + 'FlowGraphAssetContext', + initFlowGraphAssetsCtx({ getModules: () => job?.raw_flow?.modules ?? [] }) + ) + function asWorkflowStatus(x: any): Record { return x as Record } @@ -192,6 +199,7 @@ Result Logs + Assets {#if isScriptPreview(job?.job_kind)} Code {/if} @@ -205,6 +213,8 @@
+ {:else if viewTab == 'assets'} + {:else}
{#if viewTab == 'logs'} @@ -263,3 +273,8 @@
{/if} + diff --git a/frontend/src/lib/components/script_builder.ts b/frontend/src/lib/components/script_builder.ts index 6ce7377846..dd5ac52fad 100644 --- a/frontend/src/lib/components/script_builder.ts +++ b/frontend/src/lib/components/script_builder.ts @@ -1,5 +1,5 @@ import type { NewScript } from '$lib/gen' -import type { AssetWithAccessType } from './assets/lib' +import type { AssetWithAltAccessType } from './assets/lib' import type { ScriptBuilderWhitelabelCustomUi } from './custom_ui' import type { DiffDrawerI } from './diff_drawer' import type { ScriptBuilderFunctionExports } from './scriptBuilder' @@ -9,7 +9,7 @@ import type { NewScriptWithDraftAndDraftTriggers, Trigger } from './triggers/uti export interface ScriptBuilderProps { script: NewScript & { draft_triggers?: Trigger[] - fallback_access_types?: AssetWithAccessType[] + assets?: AssetWithAltAccessType[] } disableAi?: boolean fullyLoaded?: boolean diff --git a/frontend/src/lib/svelte5Utils.svelte.ts b/frontend/src/lib/svelte5Utils.svelte.ts index d45dcbfea2..d738dbf89d 100644 --- a/frontend/src/lib/svelte5Utils.svelte.ts +++ b/frontend/src/lib/svelte5Utils.svelte.ts @@ -1,5 +1,6 @@ // https://github.com/sveltejs/svelte/issues/14600 +import { untrack } from 'svelte' import type { StateStore } from './utils' export function withProps(component: Component, props: Props) { @@ -10,6 +11,11 @@ export function withProps(component: Component, props: Props) return ret } +export function createState(initialValue: T): T { + let s = $state(initialValue) + return s +} + export function stateSnapshot(state: T) { return $state.snapshot(state) } @@ -38,23 +44,25 @@ export function usePromise( status: 'loading', __promise: undefined, refresh: () => { - let promise = createPromise() - ret.__promise = promise - ret.status = 'loading' - if (clearValueOnRefresh) ret.value = undefined - ret.error = undefined + untrack(() => { + let promise = createPromise() + ret.__promise = promise + ret.status = 'loading' + if (clearValueOnRefresh) ret.value = undefined + ret.error = undefined - promise - .then((value) => { - if (ret.__promise !== promise) return - ret.value = value - ret.status = 'ok' - }) - .catch((error) => { - if (ret.__promise !== promise) return - ret.error = error - ret.status = 'error' - }) + promise + .then((value) => { + if (ret.__promise !== promise) return + ret.value = value + ret.status = 'ok' + }) + .catch((error) => { + if (ret.__promise !== promise) return + ret.error = error + ret.status = 'error' + }) + }) } }) if (loadInit) ret.refresh() diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index db01243249..12c4c3b24d 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -1465,3 +1465,20 @@ export function isS3Uri(uri: string): uri is S3Uri { const match = uri.match(/^s3:\/\/([^/]*)\/(.*)$/) return !!match && match.length === 3 } + +export function uniqueBy(array: T[], key: (t: T) => any): T[] { + const seen = new Set() + return array.filter((item) => { + const value = key(item) + if (seen.has(value)) { + return false + } else { + seen.add(value) + return true + } + }) +} + +export function pruneNullishArrayWithSet(array: (T | null | undefined)[]): T[] { + return array.filter((item): item is T => item !== null && item !== undefined) +} diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index df33996942..93e18725d7 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -60,6 +60,9 @@ import { setContext } from 'svelte' import TriggersBadge from '$lib/components/graph/renderers/triggers/TriggersBadge.svelte' import { Triggers } from '$lib/components/triggers/triggers.svelte' + import FlowAssetsHandler, { + initFlowGraphAssetsCtx + } from '$lib/components/flows/FlowAssetsHandler.svelte' let flow: Flow | undefined = $state() let can_write = false @@ -91,6 +94,11 @@ triggersState }) + setContext( + 'FlowGraphAssetContext', + initFlowGraphAssetsCtx({ getModules: () => flow?.value.modules ?? [] }) + ) + let previousPath: string | undefined = $state(undefined) async function archiveFlow(): Promise { @@ -648,3 +656,9 @@ {/snippet} {/if} + + diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 9f196b16eb..3da83d678f 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -90,8 +90,12 @@ import CustomPopover from '$lib/components/CustomPopover.svelte' import { isWindmillTooBigObject } from '$lib/components/job_args' import ScheduleEditor from '$lib/components/triggers/schedules/ScheduleEditor.svelte' - import { untrack } from 'svelte' + import { setContext, untrack } from 'svelte' import WorkerHostname from '$lib/components/WorkerHostname.svelte' + import FlowAssetsHandler, { + initFlowGraphAssetsCtx + } from '$lib/components/flows/FlowAssetsHandler.svelte' + import JobAssetsViewer from '$lib/components/assets/JobAssetsViewer.svelte' let job: Job | undefined = $state() let jobUpdateLastFetch: Date | undefined = $state() @@ -99,7 +103,7 @@ let scriptProgress: number | undefined = $state(undefined) let currentJobIsLongRunning: boolean = $state(false) - let viewTab: 'result' | 'logs' | 'code' | 'stats' = $state('result') + let viewTab: 'result' | 'logs' | 'code' | 'stats' | 'assets' = $state('result') let selectedJobStep: string | undefined = $state(undefined) let branchOrIterationN: number = $state(0) @@ -118,6 +122,12 @@ let lastJobId: string | undefined = $state(undefined) let concurrencyKey: string | undefined = $state(undefined) + + setContext( + 'FlowGraphAssetContext', + initFlowGraphAssetsCtx({ getModules: () => job?.raw_flow?.modules ?? [] }) + ) + async function getConcurrencyKey(job: Job | undefined) { if (!job) return lastJobId = job.id @@ -925,6 +935,7 @@ Result Logs Metrics + Assets {#if isScriptPreview(job?.job_kind)} Code {/if} @@ -944,6 +955,10 @@ tag={job?.tag} /> + {:else if viewTab == 'assets'} +
+ +
{:else if viewTab == 'code'} {#if job && 'raw_code' in job && job.raw_code}
@@ -1001,3 +1016,9 @@ {/if}
{/if} + + diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 54c0af034c..0de884f00a 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -270,7 +270,7 @@ components: type: string is_trigger: type: boolean - asset_fallback_access_types: + assets: type: array items: type: object @@ -287,10 +287,10 @@ components: - resource access_type: type: string - enum: - - r - - w - - rw + enum: [r, w, rw] + alt_access_type: + type: string + enum: [r, w, rw] required: - type - content