From 2b826cee5a1e58ab0e5a25f8ce0943f84d056441 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 3 Nov 2025 14:43:05 +0100 Subject: [PATCH 001/105] Hide 'show assets' toggle when there are no assets (#7037) --- frontend/src/lib/components/graph/FlowGraphV2.svelte | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index 751de1a7bc..3b41cf2035 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -530,6 +530,9 @@ expandedSubflows ) }) + let hideAssetsToggle = $derived( + $showAssets && Object.values(nodes).every((n) => n.type !== 'asset') + ) $effect(() => { ;[graph, allowSimplifiedPoll, $showAssets] @@ -672,7 +675,9 @@ class="!shadow-none gap-3" style={leftHeader ? 'margin-top: 40px;' : ''} > - + {#if !hideAssetsToggle} + + {/if} {#if showDataflow} {/if} From aaadf60d8f975df66bb9837d8fe21eaeaaf2e70b Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 3 Nov 2025 15:10:36 +0100 Subject: [PATCH 002/105] fix(backend): add 404 error when not found in resource delete endpoints (#7036) --- ...f88de5dc8ae5321fa7ff0c7632e628556f991.json | 23 +++++++++++++++++++ ...079e65a5300a52a292b9143b755087b9f9c4c.json | 23 +++++++++++++++++++ ...2acdb42868fd5f7a27f71836e7a6ff1c69856.json | 15 ------------ backend/windmill-api/src/resources.rs | 16 ++++++++----- 4 files changed, 56 insertions(+), 21 deletions(-) create mode 100644 backend/.sqlx/query-1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991.json create mode 100644 backend/.sqlx/query-85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c.json delete mode 100644 backend/.sqlx/query-d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856.json diff --git a/backend/.sqlx/query-1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991.json b/backend/.sqlx/query-1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991.json new file mode 100644 index 0000000000..4e4ae9586c --- /dev/null +++ b/backend/.sqlx/query-1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2 RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991" +} diff --git a/backend/.sqlx/query-85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c.json b/backend/.sqlx/query-85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c.json new file mode 100644 index 0000000000..5573e5208b --- /dev/null +++ b/backend/.sqlx/query-85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c" +} diff --git a/backend/.sqlx/query-d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856.json b/backend/.sqlx/query-d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856.json deleted file mode 100644 index 87361665b5..0000000000 --- a/backend/.sqlx/query-d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856" -} diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 7e6e6cb6f1..ec1eafe08a 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -798,13 +798,14 @@ async fn delete_resource( check_scopes(&authed, || format!("resources:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - "DELETE FROM resource WHERE path = $1 AND workspace_id = $2", + let deleted_path = sqlx::query_scalar!( + "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", path, w_id ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .await?; + not_found_if_none(deleted_path, "Resource", &path)?; sqlx::query!( "DELETE FROM variable WHERE path = $1 AND workspace_id = $2", path, @@ -1248,13 +1249,16 @@ async fn delete_resource_type( let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2", + let deleted_name = sqlx::query_scalar!( + "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2 RETURNING name", name, w_id ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .await?; + + not_found_if_none(deleted_name, "ResourceType", &name)?; + audit_log( &mut *tx, &authed, From 9bfd51ce0ae1926e257f76a4b72dec07b3255046 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Mon, 3 Nov 2025 15:17:06 +0100 Subject: [PATCH 003/105] fix script picker alignment (#7019) * fix default tage overflow * fix script picker alignment * fix path height * nit --- frontend/src/lib/components/DefaultTags.svelte | 5 ++++- .../flows/content/FlowInputsQuick.svelte | 18 ++++++++++-------- .../schedules/ScheduleEditorInner.svelte | 13 ++++++++++--- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/frontend/src/lib/components/DefaultTags.svelte b/frontend/src/lib/components/DefaultTags.svelte index 4884df5db4..a8ff3e2410 100644 --- a/frontend/src/lib/components/DefaultTags.svelte +++ b/frontend/src/lib/components/DefaultTags.svelte @@ -19,7 +19,10 @@ let placement: 'bottom-end' | 'top-end' = 'bottom-end' - + {#snippet trigger()} {/each} -
+
{:else} -
+
No items found.
{/if} @@ -285,7 +285,7 @@ {#if preFilter === 'hub' || preFilter === 'all'} {#if preFilter == 'all'} -
Integrations
+
Integrations
{/if} { @@ -339,8 +339,8 @@ {/if} {#if inlineScripts?.length > 0} -
-
+
New {selectedKind != 'script' ? selectedKind + ' ' : ''}script
{#if $userStore?.is_admin || $userStore?.is_super_admin} @@ -351,6 +351,7 @@ unifiedSize="sm" variant="subtle" title="Edit global default scripts" + btnClasses="-my-3" /> {:else} @@ -443,7 +445,7 @@ {#if (!selected || selected?.kind === 'owner') && (preFilter === 'workspace' || preFilter === 'all')} {#if !selected && (preFilter !== 'workspace' || funcDesc?.length > 0)} -
Workspace
+
Workspace
{/if} {#await import('../pickers/WorkspaceScriptPickerQuick.svelte') then Module} Hub
+
Hub
{/if} {#await import('../pickers/PickHubScriptQuick.svelte') then Module} - import { Alert, Badge, Button, Tab, Tabs } from '$lib/components/common' + import { Alert, Badge, Button, ButtonType, Tab, Tabs } from '$lib/components/common' import Drawer from '$lib/components/common/drawer/Drawer.svelte' import DrawerContent from '$lib/components/common/drawer/DrawerContent.svelte' import CronInput from '$lib/components/CronInput.svelte' @@ -36,6 +36,7 @@ import { runScheduleNow } from '../scheduled/utils' import { handleConfigChange } from '../utils' import TextInput from '$lib/components/text_input/TextInput.svelte' + import { twMerge } from 'tailwind-merge' let { useDrawer = true, @@ -724,7 +725,10 @@
Schedule path (not editable) @@ -733,7 +737,10 @@ readonly value={path} size={path?.length || 50} - class="font-mono !text-2xs grow shrink overflow-x-auto !py-0 !border-l-0 !rounded-l-none" + class={twMerge( + 'font-mono !text-2xs grow shrink overflow-x-auto !py-0 !border-l-0 !rounded-l-none', + ButtonType.UnifiedMinHeightClasses['md'] + )} onfocus={({ currentTarget }) => { currentTarget.select() }} From 606f8dafe72a2f5c2af6c6f4bb6727b36cbf9112 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 3 Nov 2025 15:28:37 +0100 Subject: [PATCH 004/105] chore(main): release 1.572.0 (#7027) * chore(main): release 1.572.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 20 ++++ backend/Cargo.lock | 97 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 85 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02f91aaa8a..0966f7a765 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [1.572.0](https://github.com/windmill-labs/windmill/compare/v1.571.0...v1.572.0) (2025-11-03) + + +### Features + +* **flow:** Add graph diff visualizer ([#6948](https://github.com/windmill-labs/windmill/issues/6948)) ([04d2ef4](https://github.com/windmill-labs/windmill/commit/04d2ef419dfe64c99da55ecead545cb1cc5cf185)) + + +### Bug Fixes + +* **backend:** add 404 error when not found in resource delete endpoints ([#7036](https://github.com/windmill-labs/windmill/issues/7036)) ([aaadf60](https://github.com/windmill-labs/windmill/commit/aaadf60d8f975df66bb9837d8fe21eaeaaf2e70b)) +* consider duckdb as a normal tag ([#7035](https://github.com/windmill-labs/windmill/issues/7035)) ([16317a4](https://github.com/windmill-labs/windmill/commit/16317a471446e444ada4a67d56be36b146c4dfcf)) +* fix rebuild_dependency_map ([#7026](https://github.com/windmill-labs/windmill/issues/7026)) ([f661caf](https://github.com/windmill-labs/windmill/commit/f661caf2b1bb9cb9a2f388684f94cb3055291da5)) +* include missing tags from default/native consts ([#7034](https://github.com/windmill-labs/windmill/issues/7034)) ([c04489c](https://github.com/windmill-labs/windmill/commit/c04489c463f6b277452b7480659fa65cef1ab1ba)) + + +### Performance Improvements + +* parse flow value only if needed ([#7025](https://github.com/windmill-labs/windmill/issues/7025)) ([b5e341f](https://github.com/windmill-labs/windmill/commit/b5e341fde79d6335c0e27d0adbe0febd49a09d36)) + ## [1.571.0](https://github.com/windmill-labs/windmill/compare/v1.570.0...v1.571.0) (2025-11-01) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 6a8ec39269..6963258860 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -6681,7 +6681,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tower-service", - "webpki-roots 1.0.3", + "webpki-roots 1.0.4", ] [[package]] @@ -7054,9 +7054,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344" dependencies = [ "bitflags 2.9.4", "cfg-if", @@ -8753,11 +8753,10 @@ dependencies = [ [[package]] name = "num-bigint-dig" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +checksum = "82c79c15c05d4bf82b6f5ef163104cc81a760d8e874d38ac50ab67c8877b647b" dependencies = [ - "byteorder", "lazy_static", "libm", "num-integer", @@ -10733,7 +10732,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.3", + "webpki-roots 1.0.4", ] [[package]] @@ -10883,7 +10882,7 @@ dependencies = [ "rand 0.9.0", "reqwest 0.12.24", "rmcp-macros", - "schemars 1.0.4", + "schemars 1.0.5", "serde", "serde_json", "sse-stream", @@ -11463,14 +11462,14 @@ dependencies = [ [[package]] name = "schemars" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" +checksum = "1317c3bf3e7df961da95b0a56a172a02abead31276215a0497241a7624b487ce" dependencies = [ "chrono", "dyn-clone", "ref-cast", - "schemars_derive 1.0.4", + "schemars_derive 1.0.5", "serde", "serde_json", ] @@ -11489,9 +11488,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d020396d1d138dc19f1165df7545479dcd58d93810dc5d646a16e55abefa80" +checksum = "5f760a6150d45dd66ec044983c124595ae76912e77ed0b44124cb3e415cce5d9" dependencies = [ "proc-macro2", "quote", @@ -11798,7 +11797,7 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.11.1", "schemars 0.9.0", - "schemars 1.0.4", + "schemars 1.0.5", "serde", "serde_derive", "serde_json", @@ -13819,9 +13818,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.16" +version = "0.7.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" dependencies = [ "bytes", "futures-core", @@ -14951,14 +14950,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" dependencies = [ - "webpki-root-certs 1.0.3", + "webpki-root-certs 1.0.4", ] [[package]] name = "webpki-root-certs" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d651ec480de84b762e7be71e6efa7461699c19d9e2c272c8d93455f567786e" +checksum = "ee3e3b5f5e80bc89f30ce8d0343bf4e5f12341c51f3e26cbeecbc7c85443e85b" dependencies = [ "rustls-pki-types", ] @@ -14969,14 +14968,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.3", + "webpki-roots 1.0.4", ] [[package]] name = "webpki-roots" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" dependencies = [ "rustls-pki-types", ] @@ -15138,7 +15137,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "aws-sdk-config", @@ -15198,7 +15197,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "argon2", @@ -15318,7 +15317,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.571.0" +version = "1.572.0" dependencies = [ "base64 0.22.1", "chrono", @@ -15333,7 +15332,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.571.0" +version = "1.572.0" dependencies = [ "chrono", "lazy_static", @@ -15347,7 +15346,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "axum", @@ -15366,7 +15365,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "async-recursion", @@ -15451,7 +15450,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.571.0" +version = "1.572.0" dependencies = [ "regex", "serde", @@ -15466,7 +15465,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "bytes", @@ -15490,7 +15489,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.571.0" +version = "1.572.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15502,7 +15501,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.571.0" +version = "1.572.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15511,7 +15510,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "lazy_static", @@ -15523,7 +15522,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "serde_json", @@ -15535,7 +15534,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "gosyn", @@ -15547,7 +15546,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "lazy_static", @@ -15559,7 +15558,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "serde_json", @@ -15571,7 +15570,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "nu-parser", @@ -15582,7 +15581,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15593,7 +15592,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15605,7 +15604,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "async-recursion", @@ -15628,7 +15627,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "lazy_static", @@ -15642,7 +15641,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15659,7 +15658,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "lazy_static", @@ -15673,7 +15672,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "lazy_static", @@ -15691,7 +15690,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "serde", @@ -15702,7 +15701,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "async-recursion", @@ -15735,7 +15734,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.571.0" +version = "1.572.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15745,7 +15744,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.571.0" +version = "1.572.0" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index c1ed86fc84..cd65dd72a2 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.571.0" +version = "1.572.0" authors.workspace = true edition.workspace = true @@ -33,7 +33,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.571.0" +version = "1.572.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 13a654513a..07e37caec8 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.571.0 + version: 1.572.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 173a3f4480..fa42c34827 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.571.0"; +export const VERSION = "v1.572.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 146e5792f4..624baec0e5 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.571.0"; +export const VERSION = "1.572.0"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 61855e46cc..0730eb96b7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.571.0", + "version": "1.572.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.571.0", + "version": "1.572.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 3b10f64bc8..70cf96e754 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.571.0", + "version": "1.572.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 9b086a0cc3..b709d7856a 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.571.0" -wmill_pg = ">=1.571.0" +wmill = ">=1.572.0" +wmill_pg = ">=1.572.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 34b1731352..e7c7652f67 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.571.0 + version: 1.572.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 4a87c22658..eb82effd47 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.571.0' + ModuleVersion = '1.572.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 9a5850bf23..094dbde987 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.571.0" +version = "1.572.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index eedf7a371e..61368a73dd 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.571.0" +version = "1.572.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 1aeb3c5fd8..ac0636d7c1 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.571.0", + "version": "1.572.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index b2ff911191..fb223e6a11 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.571.0", + "version": "1.572.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 4d65c69e0f..d68bd2cb6b 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.571.0 +1.572.0 From 2d1c1d81bafb7c5e551823dca6a6c66e8256fb01 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 4 Nov 2025 10:31:11 +0100 Subject: [PATCH 005/105] internal(backend): mini completed job + improvements (#7042) * mini completed job * mini completed job * sqlx * restore dedi * restore dedi * nits * fix --- ...4eafc6695257632406ed9a2111dba1dd106c7.json | 23 ++ ...4f116c4ea41ccd1bc994d59c1cd56aae71334.json | 23 -- ...26e57488489177fe596940d3698d843a93666.json | 22 -- backend/Cargo.lock | 1 + backend/ee-repo-ref.txt | 2 +- backend/src/monitor.rs | 4 +- backend/update_sqlx.sh | 5 - backend/windmill-common/src/flows.rs | 4 + backend/windmill-queue/Cargo.toml | 1 + backend/windmill-queue/src/jobs.rs | 292 ++++++++++-------- backend/windmill-worker/src/ai/tools.rs | 7 +- backend/windmill-worker/src/common.rs | 4 +- .../windmill-worker/src/dedicated_worker.rs | 16 +- .../windmill-worker/src/result_processor.rs | 15 +- backend/windmill-worker/src/worker.rs | 73 ++--- backend/windmill-worker/src/worker_flow.rs | 137 +++++++- .../lib/components/flows/DebounceLimit.svelte | 6 +- .../flows/content/FlowSettings.svelte | 14 +- 18 files changed, 393 insertions(+), 256 deletions(-) create mode 100644 backend/.sqlx/query-0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7.json delete mode 100644 backend/.sqlx/query-255415e4b1e891ce43b92ebde534f116c4ea41ccd1bc994d59c1cd56aae71334.json delete mode 100644 backend/.sqlx/query-2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666.json diff --git a/backend/.sqlx/query-0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7.json b/backend/.sqlx/query-0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7.json new file mode 100644 index 0000000000..f91485e855 --- /dev/null +++ b/backend/.sqlx/query-0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args as \"args: sqlx::types::Json>>\" FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7" +} diff --git a/backend/.sqlx/query-255415e4b1e891ce43b92ebde534f116c4ea41ccd1bc994d59c1cd56aae71334.json b/backend/.sqlx/query-255415e4b1e891ce43b92ebde534f116c4ea41ccd1bc994d59c1cd56aae71334.json deleted file mode 100644 index 3daa40769d..0000000000 --- a/backend/.sqlx/query-255415e4b1e891ce43b92ebde534f116c4ea41ccd1bc994d59c1cd56aae71334.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT f.value as \"value: Json\" FROM v2_job j JOIN flow_version f ON f.id = j.runnable_id WHERE j.id = $1 AND j.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "value: Json", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "255415e4b1e891ce43b92ebde534f116c4ea41ccd1bc994d59c1cd56aae71334" -} diff --git a/backend/.sqlx/query-2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666.json b/backend/.sqlx/query-2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666.json deleted file mode 100644 index 405902863c..0000000000 --- a/backend/.sqlx/query-2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT script_path FROM v2_as_queue WHERE id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "script_path", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - true - ] - }, - "hash": "2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666" -} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 6963258860..a0cc9e9895 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15717,6 +15717,7 @@ dependencies = [ "itertools 0.14.0", "lazy_static", "prometheus", + "quick_cache", "regex", "reqwest 0.12.24", "serde", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index dbf861025b..91e3c3f9a0 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6a3c688d51480b764d780bb02368c4b1da56caff +ba3429565e0f34c1e764389e950c9797c356ac42 \ No newline at end of file diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index fdd5cbd148..e310f0e3bf 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -80,7 +80,7 @@ use windmill_common::{ OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, }; use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING}; -use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload}; +use windmill_queue::{cancel_job, SameWorkerPayload}; use windmill_worker::{ handle_job_error, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, @@ -2338,7 +2338,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker let _ = handle_job_error( db, &client, - &MiniPulledJob::from(&job), + &windmill_queue::MiniCompletedJob::from(windmill_queue::MiniPulledJob::from(&job)), 0, None, error::Error::ExecutionErr(error_message), diff --git a/backend/update_sqlx.sh b/backend/update_sqlx.sh index 770bc760cb..5425706876 100755 --- a/backend/update_sqlx.sh +++ b/backend/update_sqlx.sh @@ -2,9 +2,6 @@ set -e -# Default directory -EE_DIR="../windmill-ee-private" - # Parse arguments while [[ "$#" -gt 0 ]]; do case $1 in @@ -14,8 +11,6 @@ while [[ "$#" -gt 0 ]]; do shift done -./substitute_ee_code.sh --dir "$EE_DIR" - # Check if running on macOS if [[ "$(uname)" == "Darwin" ]]; then echo "Running on macOS - substituting samael..." diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index cc47b0b981..bea2281766 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -527,6 +527,10 @@ impl FlowModule { .map_err(crate::error::to_anyhow) } + pub fn is_ai_agent(&self) -> bool { + self.get_type().is_ok_and(|x| x == "ai_agent") + } + pub fn is_simple(&self) -> bool { //todo: flow modules could also be simple execpt for the fact that the case of having single parallel flow approval step is not handled well (Create SuspendedTimeout) self.get_type() diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index b50773f9c8..62b8b5962b 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -45,3 +45,4 @@ axum.workspace = true serde_urlencoded.workspace = true regex.workspace = true backon.workspace = true +quick_cache.workspace = true diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 64247efd82..987025c5e7 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -13,6 +13,7 @@ use async_recursion::async_recursion; use chrono::{DateTime, Utc}; use futures::future::TryFutureExt; use itertools::Itertools; +use quick_cache::sync::Cache; #[cfg(feature = "prometheus")] use prometheus::IntCounter; use regex::Regex; @@ -33,7 +34,6 @@ use windmill_common::add_time; use windmill_common::auth::JobPerms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; -use windmill_common::flow_conversations::{add_message_to_conversation_tx, MessageType}; use windmill_common::jobs::{JobTriggerKind, EMAIL_ERROR_HANDLER_USER_EMAIL}; use windmill_common::utils::{configure_client, now_from_db}; use windmill_common::worker::{Connection, MIN_VERSION_SUPPORTS_DEBOUNCING, SCRIPT_TOKEN_EXPIRY}; @@ -137,7 +137,7 @@ pub struct CanceledBy { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct JobCompleted { - pub job: Arc, + pub job: MiniCompletedJob, pub preprocessed_args: Option>>, pub result: Arc>, pub result_columns: Option>, @@ -181,7 +181,7 @@ pub async fn cancel_single_job<'c>( .await; let add_job = add_completed_job_error( &db, - &MiniPulledJob::from(&job_running), + &MiniCompletedJob::from(MiniPulledJob::from(&job_running)), job_running.mem_peak.unwrap_or(0), Some(CanceledBy { username: Some(username.to_string()), reason: Some(reason) }), e, @@ -223,6 +223,7 @@ pub async fn cancel_job<'c>( force_cancel: bool, require_anonymous: bool, ) -> error::Result<(Transaction<'c, Postgres>, Option)> { + //TODO fetch mini completed job instead of QueuedJob let job = get_queued_job_tx(id, &w_id, &mut tx).await?; if job.is_none() { @@ -702,7 +703,7 @@ where pub async fn add_completed_job_error( db: &Pool, - queued_job: &MiniPulledJob, + completed_job: &MiniCompletedJob, mem_peak: i32, canceled_by: Option, e: serde_json::Value, @@ -713,7 +714,7 @@ pub async fn add_completed_job_error( #[cfg(feature = "prometheus")] register_metric( &WORKER_EXECUTION_FAILED, - &queued_job.tag, + &completed_job.tag, |s| { let counter = prometheus::register_int_counter!(prometheus::Opts::new( "worker_execution_failed", @@ -732,13 +733,13 @@ pub async fn add_completed_job_error( let result = WrappedError { error: e }; tracing::error!( "job {} in {} did not succeed: {}", - queued_job.id, - queued_job.workspace_id, + completed_job.id, + completed_job.workspace_id, serde_json::to_string(&result).unwrap_or_else(|_| "".to_string()) ); let _ = add_completed_job( db, - &queued_job, + &completed_job, false, false, Json(&result), @@ -757,11 +758,14 @@ pub async fn add_completed_job_error( lazy_static::lazy_static! { pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok(); pub static ref MAX_RESULT_SIZE_MB: usize = std::env::var("MAX_RESULT_SIZE_MB").unwrap_or("500".to_string()).parse().unwrap_or(500); + + // Cache for restart_unless_cancelled flag - keyed by (hash, workspace_id) + static ref RESTART_UNLESS_CANCELLED_CACHE: Cache<(i64, String), bool> = Cache::new(10000); } pub async fn add_completed_job( db: &Pool, - queued_job: &MiniPulledJob, + completed_job: &MiniCompletedJob, success: bool, skipped: bool, result: Json<&T>, @@ -787,7 +791,7 @@ pub async fn add_completed_job( let (opt_uuid, duration, _skip_downstream_error_handlers) = (|| { commit_completed_job( db, - queued_job, + completed_job, success, skipped, result, @@ -823,12 +827,12 @@ pub async fn add_completed_job( } #[cfg(feature = "cloud")] - apply_completed_job_cloud_usage(db, queued_job, duration); + apply_completed_job_cloud_usage(db, completed_job, duration); #[cfg(all(feature = "enterprise", feature = "private"))] crate::jobs_ee::apply_completed_job_error_handlers( db, - queued_job, + completed_job, success, result, &canceled_by, @@ -836,92 +840,17 @@ pub async fn add_completed_job( ) .await; - restart_job_if_perpetual(db, queued_job, &canceled_by).await?; + restart_job_if_perpetual(db, completed_job, &canceled_by).await?; - // Create assistant message if it's a flow and it's done, but only if last module is not an AI agent - if !skipped && flow_is_done { - let chat_input_enabled = queued_job.parse_chat_input_enabled(); - if chat_input_enabled.unwrap_or(false) { - // Get conversation_id from flow_status.memory_id - let flow_status = queued_job.parse_flow_status(); - let conversation_id = flow_status.and_then(|fs| fs.memory_id); - - if let Some(conversation_id) = conversation_id { - // get flow value - let parent_job = queued_job.parent_job.unwrap_or(queued_job.id); - let flow_value = sqlx::query!( - "SELECT f.value as \"value: Json\" FROM v2_job j JOIN flow_version f ON f.id = j.runnable_id WHERE j.id = $1 AND j.workspace_id = $2", - parent_job, - &queued_job.workspace_id - ) - .fetch_optional(db) - .await? - .map(|row| row.value.0); - - let last_module_is_ai_agent = flow_value - .as_ref() - .and_then(|flow_value| { - flow_value.modules.last().and_then(|m| m.get_value().ok()) - }) - .map(|v| matches!(v, FlowModuleValue::AIAgent { .. })) - .unwrap_or(false); - - // Only create assistant message if last module is NOT an AI agent, or there was an error - if !last_module_is_ai_agent || success == false { - let value = serde_json::to_value(result.0).map_err(|e| { - Error::internal_err(format!("Failed to serialize result: {e}")) - })?; - - let content = match value { - // If it's an Object with "output" key AND the output is a String, return it - serde_json::Value::Object(mut map) - if map.contains_key("output") - && matches!( - map.get("output"), - Some(serde_json::Value::String(_)) - ) => - { - if let Some(serde_json::Value::String(s)) = map.remove("output") { - s - } else { - // prettify the whole result - serde_json::to_string_pretty(&map) - .unwrap_or_else(|e| format!("Failed to serialize result: {e}")) - } - } - // Otherwise, if the whole value is a String, return it - serde_json::Value::String(s) => s, - // Otherwise, prettify the whole result - v => serde_json::to_string_pretty(&v) - .unwrap_or_else(|e| format!("Failed to serialize result: {e}")), - }; - - // Insert new assistant message - let mut tx = db.begin().await?; - add_message_to_conversation_tx( - &mut tx, - conversation_id, - Some(queued_job.id), - &content, - MessageType::Assistant, - None, - success, - ) - .await?; - tx.commit().await?; - } - } - } - } // tracing::error!("4 {:?}", start.elapsed()); - Ok((queued_job.id, duration)) + Ok((completed_job.id, duration)) } async fn commit_completed_job( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, success: bool, skipped: bool, result: Json<&T>, @@ -1250,7 +1179,7 @@ async fn commit_completed_job( tracing::info!( %job_id, root_job = ?queued_job.flow_innermost_root_job.map(|x| x.to_string()).unwrap_or_else(|| String::new()), - path = &queued_job.runnable_path(), + path = &queued_job.runnable_path, job_kind = ?queued_job.kind, started_at = ?queued_job.started_at.map(|x| x.to_string()).unwrap_or_else(|| String::new()), duration = ?duration, @@ -1271,7 +1200,7 @@ async fn commit_completed_job( async fn check_result_size( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, result: Json<&T>, ) -> Option, i64, bool), Error>> { let result_size = result.size() / 1024 / 1024; @@ -1307,7 +1236,7 @@ async fn check_result_size( async fn restart_job_if_perpetual( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, canceled_by: &Option, ) -> Result<(), Error> { if !queued_job.is_flow_step() && queued_job.kind == JobKind::Script && canceled_by.is_none() { @@ -1333,18 +1262,27 @@ async fn restart_job_if_perpetual( async fn restart_job_if_perpetual_inner( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, hash: ScriptHash, ) -> Result<(), Error> { - let restart = sqlx::query_scalar!( - "SELECT restart_unless_cancelled FROM script WHERE hash = $1 AND workspace_id = $2", - hash.0, - &queued_job.workspace_id - ) - .fetch_optional(db) - .await? - .flatten() - .unwrap_or(false); + let cache_key = (hash.0, queued_job.workspace_id.clone()); + + let restart = if let Some(cached) = RESTART_UNLESS_CANCELLED_CACHE.get(&cache_key) { + cached + } else { + let restart = sqlx::query_scalar!( + "SELECT restart_unless_cancelled FROM script WHERE hash = $1 AND workspace_id = $2", + hash.0, + &queued_job.workspace_id + ) + .fetch_optional(db) + .await? + .flatten() + .unwrap_or(false); + + RESTART_UNLESS_CANCELLED_CACHE.insert(cache_key, restart); + restart + }; if restart { let tx = PushIsolationLevel::IsolatedRoot(db.clone()); @@ -1364,17 +1302,25 @@ async fn restart_job_if_perpetual_inner( None }; - let ehm = HashMap::new(); + let args = sqlx::query_scalar!( + "SELECT args as \"args: sqlx::types::Json>>\" FROM v2_job WHERE id = $1 AND workspace_id = $2", + queued_job.id, + queued_job.workspace_id + ) + .fetch_optional(db) + .await? + .flatten() + .unwrap_or_default(); let (_uuid, tx) = push( db, tx, &queued_job.workspace_id, JobPayload::ScriptHash { hash, - path: queued_job.runnable_path().to_string(), + path: queued_job.runnable_path.clone().unwrap_or_default(), custom_concurrency_key: custom_concurrency_key(db, &queued_job.id).await?, - concurrent_limit: queued_job.concurrent_limit, - concurrency_time_window_s: queued_job.concurrency_time_window_s, + concurrent_limit: None, + concurrency_time_window_s: None, cache_ttl: queued_job.cache_ttl, dedicated_worker: None, language: queued_job @@ -1387,11 +1333,7 @@ async fn restart_job_if_perpetual_inner( custom_debounce_key: None, debounce_delay_s: None, }, - queued_job - .args - .as_ref() - .map(|x| PushArgs::from(&x.0)) - .unwrap_or_else(|| PushArgs::from(&ehm)), + PushArgs::from(&args.0), &queued_job.created_by, &queued_job.permissioned_as_email, queued_job.permissioned_as.clone(), @@ -1405,9 +1347,9 @@ async fn restart_job_if_perpetual_inner( false, false, None, - queued_job.visible_to_owner, + true, Some(queued_job.tag.clone()), - queued_job.timeout, + None, None, queued_job.priority, None, @@ -1424,7 +1366,7 @@ async fn restart_job_if_perpetual_inner( #[cfg(feature = "cloud")] fn apply_completed_job_cloud_usage( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, _duration: i64, ) { if *CLOUD_HOSTED && !queued_job.is_flow() && _duration > 1000 { @@ -1489,7 +1431,7 @@ fn apply_completed_job_cloud_usage( } pub async fn send_error_to_global_handler<'a, T: Serialize + Send + Sync>( - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, db: &Pool, result: Json<&T>, ) -> Result<(), Error> { @@ -1525,7 +1467,7 @@ pub async fn send_error_to_global_handler<'a, T: Serialize + Send + Sync>( } pub async fn report_error_to_workspace_handler_or_critical_side_channel( - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, db: &Pool, error_message: String, ) -> () { @@ -1586,8 +1528,9 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel( } } +//TODO cache all values pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>( - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, is_canceled: bool, db: &Pool, result: Json<&'a T>, @@ -1670,7 +1613,7 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> pub async fn handle_maybe_scheduled_job<'c>( db: &Pool, - job: &MiniPulledJob, + job: &MiniCompletedJob, schedule: &Schedule, script_path: &str, w_id: &str, @@ -1978,6 +1921,110 @@ pub struct MiniPulledJob { pub permissioned_as_end_user_email: Option, } + +#[derive(Debug, Clone, Serialize, Deserialize)] + +pub struct MiniCompletedJob { + pub id: Uuid, + pub workspace_id: String, + pub runnable_id: Option, + pub scheduled_for: chrono::DateTime, + pub parent_job: Option, + // pub root_job: Option, + pub flow_innermost_root_job: Option, + pub runnable_path: Option, + pub kind: JobKind, + pub started_at: Option>, + pub permissioned_as: String, + pub created_by: String, + pub script_lang: Option, + pub permissioned_as_email: String, + pub flow_step_id: Option, + pub trigger_kind: Option, + pub trigger: Option, + pub priority: Option, + pub concurrent_limit: Option, + pub tag: String, + pub cache_ttl: Option, +} + +impl From for MiniCompletedJob { + fn from(job: MiniPulledJob) -> Self { + MiniCompletedJob { + id: job.id, + workspace_id: job.workspace_id, + runnable_id: job.runnable_id, + scheduled_for: job.scheduled_for, + parent_job: job.parent_job, + // root_job: job.root_job,, + flow_innermost_root_job: job.flow_innermost_root_job, + runnable_path: job.runnable_path, + kind: job.kind, + started_at: job.started_at, + permissioned_as: job.permissioned_as, + created_by: job.created_by, + script_lang: job.script_lang, + permissioned_as_email: job.permissioned_as_email, + flow_step_id: job.flow_step_id, + trigger_kind: job.trigger_kind, + trigger: job.trigger, + priority: job.priority, + concurrent_limit: job.concurrent_limit, + tag: job.tag, + cache_ttl: job.cache_ttl, + } + } +} + +impl From> for MiniCompletedJob { + fn from(job: Arc) -> Self { + MiniCompletedJob { + id: job.id, + workspace_id: job.workspace_id.clone(), + runnable_id: job.runnable_id, + scheduled_for: job.scheduled_for, + parent_job: job.parent_job, + flow_innermost_root_job: job.flow_innermost_root_job, + runnable_path: job.runnable_path.clone(), + kind: job.kind, + started_at: job.started_at, + permissioned_as: job.permissioned_as.clone(), + created_by: job.created_by.clone(), + script_lang: job.script_lang, + permissioned_as_email: job.permissioned_as_email.clone(), + flow_step_id: job.flow_step_id.clone(), + trigger_kind: job.trigger_kind.clone(), + trigger: job.trigger.clone(), + priority: job.priority, + concurrent_limit: job.concurrent_limit, + tag: job.tag.clone(), + cache_ttl: job.cache_ttl, + } + } +} + +impl MiniCompletedJob { + pub fn is_flow_step(&self) -> bool { + self.flow_step_id.is_some() + } + pub fn schedule_path(&self) -> Option { + if self.trigger_kind.as_ref().is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) { + self.trigger.clone() + } else { + None + } + } + + pub fn is_flow(&self) -> bool { + self.kind.is_flow() + } + + pub fn is_dependency(&self) -> bool { + self.kind.is_dependency() + } + +} + #[derive(Serialize, Deserialize, Debug, Clone)] struct FlowStatusChatInputEnabled { chat_input_enabled: Option, @@ -2286,7 +2333,7 @@ impl PulledJobResult { PulledJobResult { job: Some(job), missing_concurrency_key: true, .. } => Err( PulledJobResultToJobErr::MissingConcurrencyKey(JobCompleted { preprocessed_args: None, - job: Arc::new(job.job), + job: MiniCompletedJob::from(job.job), success: false, result: Arc::new(windmill_common::worker::to_raw_value(&json!({ "name": "InternalErr", @@ -4723,11 +4770,6 @@ pub async fn push<'c, 'd>( ); } }; - #[cfg(not(all(feature = "enterprise", feature = "private")))] - { - let (_, _) = (debounce_delay_s, custom_debounce_key); - scheduled_for_o = scheduled_for_o; - } #[cfg(all(feature = "enterprise", feature = "private"))] if schedule_path.is_none() { diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index fc45ed08cc..d1280f0053 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -33,7 +33,8 @@ use windmill_common::{ worker::{to_raw_value, Connection}, }; use windmill_queue::{ - get_mini_pulled_job, push, JobCompleted, MiniPulledJob, PushArgs, PushIsolationLevel, + get_mini_pulled_job, push, JobCompleted, MiniCompletedJob, MiniPulledJob, PushArgs, + PushIsolationLevel, }; /// Context for tool execution containing all required references and state @@ -545,7 +546,7 @@ async fn execute_windmill_tool( ctx, tool_call, tool_module, - &tool_job, + &MiniCompletedJob::from(tool_job), job_id, err, messages, @@ -576,7 +577,7 @@ async fn handle_tool_execution_error( ctx: &mut ToolExecutionContext<'_>, tool_call: &OpenAIToolCall, tool_module: &windmill_common::flows::FlowModule, - tool_job: &MiniPulledJob, + tool_job: &MiniCompletedJob, job_id: Uuid, err: Error, messages: &mut Vec, diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index bcc529258d..892a3c8b7e 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -34,7 +34,7 @@ use windmill_common::{ use anyhow::{anyhow, Result}; use windmill_parser_sql::{s3_mode_extension, S3ModeArgs, S3ModeFormat}; -use windmill_queue::MiniPulledJob; +use windmill_queue::{MiniCompletedJob, MiniPulledJob}; use std::collections::HashSet; use std::path::Path; @@ -973,7 +973,7 @@ pub async fn get_cached_resource_value_if_valid( pub async fn save_in_cache( db: &Pool, _client: &AuthedClient, - job: &MiniPulledJob, + job: &MiniCompletedJob, cached_path: String, r: Arc>, ) { diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index 6ce2717d38..e65655c94d 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -76,7 +76,7 @@ pub async fn handle_dedicated_process( ) -> std::result::Result<(), error::Error> { //do not cache local dependencies - use windmill_queue::{JobCompleted, MiniPulledJob}; + use windmill_queue::{JobCompleted, MiniCompletedJob}; use crate::{handle_child::process_status, PROXY_ENVS}; let cmd_name = format!("dedicated {command_path}"); @@ -133,8 +133,7 @@ pub async fn handle_dedicated_process( } }); - let mut jobs: VecDeque> = - VecDeque::with_capacity(MAX_BUFFERED_DEDICATED_JOBS); + let mut jobs: VecDeque = VecDeque::with_capacity(MAX_BUFFERED_DEDICATED_JOBS); // let mut i = 0; // let mut j = 0; let mut alive = true; @@ -179,7 +178,7 @@ pub async fn handle_dedicated_process( } tracing::debug!("processed job: |{line}|"); if line.starts_with("wm_res[") { - let job: Arc = jobs.pop_front().expect("pop"); + let job = jobs.pop_front().expect("pop"); tracing::info!("job completed on dedicated worker {script_path}: {}", job.id); match serde_json::from_str::>(&line.replace("wm_res[success]:", "").replace("wm_res[error]:", "")) { Ok(result) => { @@ -215,12 +214,15 @@ pub async fn handle_dedicated_process( job = conditional_polling(jobs_rx.recv(), alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS) => { // i += 1; if let Some(job) = job { - jobs.push_back(job.clone()); - tracing::info!("received job and adding to queue on dedicated worker for {script_path}: {} (queue_size: {})", job.id, jobs.len()); + let id = job.id; + let args = serde_json::to_string(&job.args).expect("serialize"); + jobs.push_back(MiniCompletedJob::from(job)); + tracing::info!("received job and adding to queue on dedicated worker for {script_path}: {} (queue_size: {})", id, jobs.len()); // write_stdin(&mut stdin, &serde_json::to_string(&job.args.unwrap_or_else(|| serde_json::json!({"x": job.id}))).expect("serialize")).await?; - write_stdin(&mut stdin, &serde_json::to_string(&job.args).expect("serialize")).await?; + write_stdin(&mut stdin, &args).await?; stdin.flush().await.context("stdin flush")?; + // tracing::info!("wrote job to stdin for {script_path}: {} (queue_size: {})", id, jobs.len()); } else { tracing::debug!("job channel closed"); alive = false; diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 9f4fd7907d..505213bb68 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -31,8 +31,8 @@ use windmill_common::{ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ - append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, ValidableJson, - WrappedError, INIT_SCRIPT_TAG, + append_logs, get_queued_job, CanceledBy, JobCompleted, MiniCompletedJob, MiniPulledJob, + ValidableJson, WrappedError, INIT_SCRIPT_TAG, }; use serde_json::{json, value::RawValue, Value}; @@ -400,7 +400,7 @@ async fn send_job_completed(job_completed_tx: JobCompletedSender, jc: JobComplet } pub async fn process_result( - job: Arc, + job: MiniCompletedJob, result: error::Result>>, job_dir: &str, job_completed_tx: JobCompletedSender, @@ -539,7 +539,7 @@ pub async fn handle_receive_completed_job( handle_job_error( db, &client, - job.as_ref(), + &job, mem_peak, canceled_by, err, @@ -721,7 +721,7 @@ pub async fn process_completed_job( pub async fn handle_non_flow_job_error( db: &DB, - job: &MiniPulledJob, + job: &MiniCompletedJob, mem_peak: i32, canceled_by: Option, err_string: String, @@ -752,7 +752,7 @@ pub async fn handle_non_flow_job_error( pub async fn handle_job_error( db: &DB, client: &AuthedClient, - job: &MiniPulledJob, + job: &MiniCompletedJob, mem_peak: i32, canceled_by: Option, err: Error, @@ -816,6 +816,7 @@ pub async fn handle_job_error( if let Err(err) = updated_flow { if let Some(parent_job_id) = job.parent_job { + // TODO get minicompleted job directly if let Ok(Some(parent_job)) = get_queued_job(&parent_job_id, &job.workspace_id, &db).await { @@ -829,7 +830,7 @@ pub async fn handle_job_error( .await; let _ = add_completed_job_error( db, - &MiniPulledJob::from(&parent_job), + &MiniCompletedJob::from(MiniPulledJob::from(&parent_job)), mem_peak, canceled_by.clone(), e, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 0e6da36a20..8cf1e8a7be 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -59,6 +59,7 @@ use std::{ }; use windmill_parser::MainArgSignature; use windmill_queue::preprocess_dependency_job; +use windmill_queue::MiniCompletedJob; use windmill_queue::PulledJobResultToJobErr; use uuid::Uuid; @@ -741,7 +742,7 @@ fn create_span(arc_job: &Arc, worker_name: &str, hostname: &str) pub async fn handle_all_job_kind_error( conn: &Connection, authed_client: &AuthedClient, - job: Arc, + job: MiniCompletedJob, err: Error, same_worker_tx: Option<&SameWorkerSender>, worker_dir: &str, @@ -754,7 +755,7 @@ pub async fn handle_all_job_kind_error( handle_job_error( db, authed_client, - job.as_ref(), + &job, 0, None, err, @@ -773,7 +774,7 @@ pub async fn handle_all_job_kind_error( .send_job( JobCompleted { preprocessed_args: None, - job: job.clone(), + job: job, result: Arc::new(windmill_common::worker::to_raw_value(&error_to_value( &err, ))), @@ -954,8 +955,6 @@ pub async fn run_worker( killpill_tx: KillpillSender, base_internal_url: &str, ) { - - #[cfg(not(feature = "enterprise"))] if !*DISABLE_NSJAIL { tracing::warn!( @@ -1749,7 +1748,7 @@ pub async fn run_worker( .send_job( JobCompleted { preprocessed_args: None, - job: Arc::new(job.job()), + job: MiniCompletedJob::from(job.job()), success: true, result: Arc::new(empty_result()), result_columns: None, @@ -1975,7 +1974,7 @@ pub async fn run_worker( handle_all_job_kind_error( &conn, &authed_client, - arc_job.clone(), + MiniCompletedJob::from(arc_job), err, Some(&same_worker_tx), &worker_dir, @@ -2364,7 +2363,6 @@ pub async fn handle_queued_job( precomputed_agent_info: Option, #[cfg(feature = "benchmark")] _bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { - // Extract the active span from the context if job.canceled_by.is_some() { @@ -2437,14 +2435,8 @@ pub async fn handle_queued_job( ) => { if x.map(|x| x.0).is_none_or(|x| is_special_codebase_hash(x)) { Some( - cache::job::fetch_preview( - conn, - &job.id, - raw_lock, - raw_code, - raw_flow.clone(), - ) - .await?, + cache::job::fetch_preview(conn, &job.id, raw_lock, raw_code, raw_flow.clone()) + .await?, ) } else { None @@ -2485,7 +2477,7 @@ pub async fn handle_queued_job( .send_job( JobCompleted { preprocessed_args: None, - job, + job: MiniCompletedJob::from(job), result, result_columns: None, mem_peak: 0, @@ -2506,10 +2498,7 @@ pub async fn handle_queued_job( tracing::debug!("Send job completed") } Err(err) => { - tracing::error!( - "An error occurred while sending job completed: {:#?}", - err - ) + tracing::error!("An error occurred while sending job completed: {:#?}", err) } } @@ -2544,7 +2533,6 @@ pub async fn handle_queued_job( )); } } else { - let mut logs = "".to_string(); let mut mem_peak: i32 = 0; let mut canceled_by: Option = None; @@ -2729,7 +2717,8 @@ pub async fn handle_queued_job( killpill_rx, precomputed_agent_info, &mut has_stream, - )).await; + )) + .await; occupancy_metrics.total_duration_of_running_jobs += metric_timer.elapsed().as_secs_f32(); @@ -2737,8 +2726,10 @@ pub async fn handle_queued_job( } }; + let cjob = MiniCompletedJob::from(job.to_owned()); + drop(job); //it's a test job, no need to update the db - if job.as_ref().workspace_id == "" { + if cjob.workspace_id == "" { return Ok(true); } @@ -2749,7 +2740,7 @@ pub async fn handle_queued_job( return Ok(false); } process_result( - job, + cjob, result.map(|x| Arc::new(x)), job_dir, job_completed_tx, @@ -2765,8 +2756,6 @@ pub async fn handle_queued_job( ) .await } - - } pub fn build_envs( @@ -2953,7 +2942,6 @@ async fn handle_code_execution_job( precomputed_agent_info: Option, has_stream: &mut bool, ) -> error::Result> { - let script_hash = || { job.runnable_id .ok_or_else(|| Error::internal_err("expected script hash")) @@ -2994,8 +2982,11 @@ async fn handle_code_execution_job( } JobKind::Script_Hub => { let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema } = - Box::pin(get_hub_script_content_and_requirements(job.runnable_path.as_ref(), conn.as_sql())) - .await?; + Box::pin(get_hub_script_content_and_requirements( + job.runnable_path.as_ref(), + conn.as_sql(), + )) + .await?; data = ScriptData { code: content, lock: lockfile }; metadata = ScriptMetadata { language, envs, codebase, schema, schema_validator: None }; @@ -3006,7 +2997,11 @@ async fn handle_code_execution_job( (arc_data.as_ref(), arc_metadata.as_ref()) } JobKind::FlowScript => { - arc_data = Box::pin(cache::flow::fetch_script(conn, FlowNodeId(script_hash()?.0))).await?; + arc_data = Box::pin(cache::flow::fetch_script( + conn, + FlowNodeId(script_hash()?.0), + )) + .await?; metadata = ScriptMetadata { language: job.script_lang, envs: None, @@ -3017,7 +3012,11 @@ async fn handle_code_execution_job( (arc_data.as_ref(), &metadata) } JobKind::AppScript => { - arc_data = Box::pin(cache::app::fetch_script(conn, AppScriptId(script_hash()?.0))).await?; + arc_data = Box::pin(cache::app::fetch_script( + conn, + AppScriptId(script_hash()?.0), + )) + .await?; metadata = ScriptMetadata { language: job.script_lang, envs: None, @@ -3035,8 +3034,11 @@ async fn handle_code_execution_job( .ok_or_else(|| Error::internal_err("expected script path".to_string()))?; if script_path.starts_with("hub/") { let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema } = - Box::pin(get_hub_script_content_and_requirements(Some(script_path), conn.as_sql())) - .await?; + Box::pin(get_hub_script_content_and_requirements( + Some(script_path), + conn.as_sql(), + )) + .await?; data = ScriptData { code: content, lock: lockfile }; metadata = ScriptMetadata { language, envs, codebase, schema, schema_validator: None }; @@ -3052,7 +3054,8 @@ async fn handle_code_execution_job( .await? .ok_or_else(|| Error::internal_err("expected script hash".to_string()))?; - (arc_data, arc_metadata) = Box::pin(cache::script::fetch(conn, ScriptHash(hash))).await?; + (arc_data, arc_metadata) = + Box::pin(cache::script::fetch(conn, ScriptHash(hash))).await?; (arc_data.as_ref(), arc_metadata.as_ref()) } } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 662c947842..dd34f4dc24 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -36,6 +36,7 @@ use windmill_common::bench::BenchmarkIter; use windmill_common::cache::{self, RawData}; use windmill_common::client::AuthedClient; use windmill_common::db::Authed; +use windmill_common::flow_conversations::{add_message_to_conversation_tx, MessageType}; use windmill_common::flow_status::{ ApprovalConditions, FlowJobDuration, FlowJobsDuration, FlowStatusModuleWParent, Iterator as FlowIterator, JobResult, @@ -64,7 +65,7 @@ use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, handle_maybe_scheduled_job, insert_concurrency_key, interpolate_args, CanceledBy, - MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, + MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, }; use windmill_audit::audit_oss::{audit_log, AuditAuthor}; @@ -287,6 +288,11 @@ pub async fn update_flow_status_after_job_completion_internal( ) -> error::Result { let mut has_triggered_error_handler = has_triggered_error_handler; add_time!(bench, "update flow status internal START"); + struct ChatAiInfo { + chat_input_enabled: bool, + conversation_id: Option, + is_ai_agent_step: bool, + } let ( should_continue_flow, flow_job, @@ -296,6 +302,7 @@ pub async fn update_flow_status_after_job_completion_internal( nresult, is_failure_step, _cleanup_module, + chat_ai_info, ) = { // tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}"); @@ -1359,6 +1366,11 @@ pub async fn update_flow_status_after_job_completion_internal( current_module_id = %current_module.map(|x| x.id.clone()).unwrap_or_default(), continue_on_error = %continue_on_error, should_continue_flow = %should_continue_flow, "computed if flow should continue"); + let chat_ai_info = ChatAiInfo { + chat_input_enabled: old_status.chat_input_enabled.unwrap_or(false), + conversation_id: old_status.memory_id, + is_ai_agent_step: current_module.is_some_and(|m| m.is_ai_agent()), + }; ( should_continue_flow, flow_job, @@ -1368,6 +1380,7 @@ pub async fn update_flow_status_after_job_completion_internal( nresult, is_failure_step, old_status.cleanup_module, + chat_ai_info, ) }; @@ -1437,37 +1450,61 @@ pub async fn update_flow_status_after_job_completion_internal( } if flow_job.is_canceled() { + let canceled_by = CanceledBy { + username: flow_job.canceled_by.clone(), + reason: flow_job.canceled_reason.clone(), + }; + let error = canceled_job_to_result(&flow_job); add_completed_job_error( db, - &flow_job, + &MiniCompletedJob::from(flow_job.clone()), 0, - Some(CanceledBy { - username: flow_job.canceled_by.clone(), - reason: flow_job.canceled_reason.clone(), - }), - canceled_job_to_result(&flow_job), + Some(canceled_by), + error, worker_name, true, None, ) .await?; } else { + let cflow_job: MiniCompletedJob = MiniCompletedJob::from(flow_job.clone()); + if flow_job.cache_ttl.is_some() && success { let flow = RawData::Flow(flow_data.clone()); let cached_res_path = cached_result_path(db, client, &flow_job, Some(&flow)).await; - save_in_cache(db, client, &flow_job, cached_res_path, nresult.clone()).await; + save_in_cache( + db, + client, + &MiniCompletedJob::from(cflow_job.clone()), + cached_res_path, + nresult.clone(), + ) + .await; } let success = success && (!is_failure_step || result_has_recover_true(nresult.clone())); add_time!(bench, "flow status update 1"); + + let skipped = stop_early && skip_if_stop_early; + add_tool_message_to_conversation( + db, + &job_id_for_status, + success, + skipped, + chat_ai_info.is_ai_agent_step, + &nresult, + chat_ai_info.chat_input_enabled, + chat_ai_info.conversation_id, + ) + .await?; let duration = if success { let (_, duration) = add_completed_job( db, - &flow_job, + &cflow_job, true, - stop_early && skip_if_stop_early, + skipped, Json(&nresult), None, 0, @@ -1482,9 +1519,9 @@ pub async fn update_flow_status_after_job_completion_internal( } else { let (_, duration) = add_completed_job( db, - &flow_job, + &cflow_job, false, - stop_early && skip_if_stop_early, + skipped, Json( &serde_json::from_str::(nresult.get()).unwrap_or_else( |e| json!({"error": format!("Impossible to serialize error: {e:#}")}), @@ -1531,8 +1568,17 @@ pub async fn update_flow_status_after_job_completion_internal( &db.into(), ) .await; - let _ = add_completed_job_error(db, &flow_job, 0, None, e, worker_name, true, None) - .await; + let _ = add_completed_job_error( + db, + &MiniCompletedJob::from(flow_job.clone()), + 0, + None, + e, + worker_name, + true, + None, + ) + .await; true } Ok(_) => false, @@ -1575,6 +1621,67 @@ fn find_flow_job_index(flow_jobs: &Vec, job_id_for_status: &Uuid) -> Optio flow_jobs.iter().position(|x| x == job_id_for_status) } +async fn add_tool_message_to_conversation( + db: &DB, + job_id: &Uuid, + success: bool, + skipped: bool, + is_ai_agent_step: bool, + result: &Box, + chat_input_enabled: bool, + conversation_id: Option, +) -> error::Result<()> { + // Create assistant message if it's a flow and it's done, but only if last module is not an AI agent + if !skipped && chat_input_enabled { + // Get conversation_id from flow_status.memory_id + + if let Some(conversation_id) = conversation_id { + // Only create assistant message if last module is NOT an AI agent, or there was an error + if !is_ai_agent_step || success == false { + let value = serde_json::to_value(result.get()) + .map_err(|e| Error::internal_err(format!("Failed to serialize result: {e}")))?; + + let content = match value { + // If it's an Object with "output" key AND the output is a String, return it + serde_json::Value::Object(mut map) + if map.contains_key("output") + && matches!(map.get("output"), Some(serde_json::Value::String(_))) => + { + if let Some(serde_json::Value::String(s)) = map.remove("output") { + s + } else { + // prettify the whole result + serde_json::to_string_pretty(&map) + .unwrap_or_else(|e| format!("Failed to serialize result: {e}")) + } + } + // Otherwise, if the whole value is a String, return it + serde_json::Value::String(s) => s, + // Otherwise, prettify the whole result + v => serde_json::to_string_pretty(&v) + .unwrap_or_else(|e| format!("Failed to serialize result: {e}")), + }; + + // Insert new assistant message + let mut tx = db.begin().await?; + add_message_to_conversation_tx( + &mut tx, + conversation_id, + Some(job_id.clone()), + &content, + MessageType::Assistant, + None, + success, + ) + .await?; + tx.commit().await?; + } + } + } + + Ok(()) +} + async fn set_success_and_duration_in_flow_job_success<'c>( flow_jobs_success: &Option>>, flow_jobs: &Vec, @@ -1939,7 +2046,7 @@ pub async fn handle_flow( if let Some(schedule) = schedule { if let Err(err) = handle_maybe_scheduled_job( db, - &flow_job, + &MiniCompletedJob::from(flow_job.clone()), &schedule, flow_job.runnable_path.as_ref().unwrap(), &flow_job.workspace_id, diff --git a/frontend/src/lib/components/flows/DebounceLimit.svelte b/frontend/src/lib/components/flows/DebounceLimit.svelte index 19d8b30f75..c35a8981d8 100644 --- a/frontend/src/lib/components/flows/DebounceLimit.svelte +++ b/frontend/src/lib/components/flows/DebounceLimit.svelte @@ -10,20 +10,22 @@ debounce_key = $bindable(), placeholder, size = 'xs', - color = undefined + color = undefined, + fontClass = 'font-normal' }: { debounce_delay_s: number | undefined debounce_key: string | undefined placeholder: string size: 'xs' | 'sm' color?: 'nord' | undefined + fontClass?: string } = $props()
- {#if flowStore.val.schema && enableAi} - - {/if} {#if customUi?.settingsTabs?.workerGroup != false}
@@ -478,6 +472,7 @@ {/if}
-
+ {#if flowStore.val.schema && enableAi} + + {/if}
From 4cfaa19bad018a53a50d5373b49db65fdf2f1da1 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 4 Nov 2025 10:14:55 +0000 Subject: [PATCH 006/105] nit --- backend/windmill-queue/src/jobs.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 987025c5e7..4311972923 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -3452,6 +3452,7 @@ pub async fn push<'c, 'd>( mut email: &str, mut permissioned_as: String, token_prefix: Option<&str>, + #[allow(unused_mut)] mut scheduled_for_o: Option>, schedule_path: Option, parent_job: Option, @@ -3654,6 +3655,7 @@ pub async fn push<'c, 'd>( } let mut preprocessed = None; + #[allow(unused)] let ( script_hash, script_path, From e6d700878feaa8f473c8da86a539696fb710395c Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Tue, 4 Nov 2025 12:36:04 +0100 Subject: [PATCH 007/105] fix typo (#7044) --- backend/windmill-common/src/flows.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index bea2281766..09861d6b10 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -528,7 +528,7 @@ impl FlowModule { } pub fn is_ai_agent(&self) -> bool { - self.get_type().is_ok_and(|x| x == "ai_agent") + self.get_type().is_ok_and(|x| x == "aiagent") } pub fn is_simple(&self) -> bool { From 5da494b19776ff55e9592a9ac88c53f028848632 Mon Sep 17 00:00:00 2001 From: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> Date: Tue, 4 Nov 2025 20:08:02 +0700 Subject: [PATCH 008/105] refactor: remove legacy database views v2_as_queue and v2_as_completed_job (#6689) * refactor: remove legacy database views v2_as_queue and v2_as_completed_job Signed-off-by: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> * fix tests * fix jobs.rs * end * fix * improvement * improvement --------- Signed-off-by: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> Co-authored-by: Ruben Fiszel --- ...af8688a6d1643747be3ec4f784c3029a59e52.json | 22 ++ ...2849fe6bc668654ffcbfbc22a02027280739.json} | 4 +- ...1e9366d1f6d3d2aa485e0f50c6f2d85693dd.json} | 6 +- ...84846f74d468f0073f02f81122895e86c364.json} | 4 +- ...bbd75d6d5bb77e6a92bf187d978a059d7af4a.json | 20 ++ ...f0563e3f98fff1f100616c33e8dc95fbff99.json} | 6 +- ...7abdad3198d3340ec7c04ed671baff0a4d0b.json} | 6 +- ...52ff982821ed7b24574cdd09a06eac0d628b.json} | 4 +- ...a2611dab30faff2891fc3a7f00ee8c120950.json} | 6 +- ...44744910a17af23f8201d853f0a4f8404fc73.json | 29 +++ ...64c72fe4b3cf2497644d514c5c00d6d71bf3.json} | 8 +- ...d5d3309a31b195b3147fa32f2ca8a6c9c90e.json} | 6 +- ...92ef0ad30e39eec59c27bae8cb0622062c8fb.json | 20 -- ...84028ce8a077308979a5ba8ef252e19aa825.json} | 4 +- ...13776870612e8faa8d8fa577f8e7b7309f76.json} | 10 +- ...2a6d9a6393c6bb1783ab8903d87dd099e236b.json | 58 +++++ ...aa01a048fa4f9281327cf5b78111178424b43.json | 58 ----- ...81a4f2edad152581950fdd80d758a0d242c17.json | 28 -- ...e5a3d1a91ea02de7608676575e1c03023ed71.json | 29 --- ...506dce6eb4af3fb32e6a4c6e76084873a539a.json | 216 ++++++++++++++++ ...dbb17c739415850e7c67f8575fb983c295fe.json} | 4 +- ...84b12dd961ba732772d8f6c59d18fe05f285d.json | 24 ++ ...ca3516ac5c1d32a98946196b1a42e3f103efd.json | 239 ++++++++++++++++++ ...1f78d7d61fbf83dda9b39bb22e0f9584d221b.json | 24 -- ...4a8b793c4ae618a04220b3609c8c3c168f8fd.json | 28 ++ ...7fd4280466eec443aff4f1e8c1ab810cca7e.json} | 4 +- backend/src/monitor.rs | 75 +++--- backend/tests/common/mod.rs | 12 +- backend/tests/worker.rs | 8 +- backend/windmill-api/src/approvals.rs | 44 ++-- backend/windmill-api/src/apps.rs | 14 +- backend/windmill-api/src/jobs.rs | 156 +++--------- backend/windmill-api/src/users.rs | 15 +- backend/windmill-queue/src/jobs.rs | 205 ++++++++++----- .../windmill-worker/src/result_processor.rs | 8 +- 35 files changed, 933 insertions(+), 471 deletions(-) create mode 100644 backend/.sqlx/query-0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52.json rename backend/.sqlx/{query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json => query-1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739.json} (63%) rename backend/.sqlx/{query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json => query-24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd.json} (52%) rename backend/.sqlx/{query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json => query-25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364.json} (50%) create mode 100644 backend/.sqlx/query-2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a.json rename backend/.sqlx/{query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json => query-29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99.json} (58%) rename backend/.sqlx/{query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json => query-35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b.json} (54%) rename backend/.sqlx/{query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json => query-371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b.json} (64%) rename backend/.sqlx/{query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json => query-37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950.json} (55%) create mode 100644 backend/.sqlx/query-4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73.json rename backend/.sqlx/{query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json => query-48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3.json} (73%) rename backend/.sqlx/{query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json => query-49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e.json} (65%) delete mode 100644 backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json rename backend/.sqlx/{query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json => query-57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825.json} (50%) rename backend/.sqlx/{query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json => query-6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76.json} (57%) create mode 100644 backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json delete mode 100644 backend/.sqlx/query-72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43.json delete mode 100644 backend/.sqlx/query-89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17.json delete mode 100644 backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json create mode 100644 backend/.sqlx/query-a001b4254e0f1ba8a87776e32f9506dce6eb4af3fb32e6a4c6e76084873a539a.json rename backend/.sqlx/{query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json => query-a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe.json} (55%) create mode 100644 backend/.sqlx/query-a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d.json create mode 100644 backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json delete mode 100644 backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json create mode 100644 backend/.sqlx/query-edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd.json rename backend/.sqlx/{query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json => query-fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e.json} (52%) diff --git a/backend/.sqlx/query-0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52.json b/backend/.sqlx/query-0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52.json new file mode 100644 index 0000000000..1880104006 --- /dev/null +++ b/backend/.sqlx/query-0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.id\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE r.ping < now() - ($1 || ' seconds')::interval\n AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52" +} diff --git a/backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json b/backend/.sqlx/query-1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739.json similarity index 63% rename from backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json rename to backend/.sqlx/query-1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739.json index ef26a8453e..4dac49a78c 100644 --- a/backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json +++ b/backend/.sqlx/query-1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) FROM v2_as_queue WHERE email = $1", + "query": "SELECT COUNT(id) FROM v2_job WHERE permissioned_as_email = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c" + "hash": "1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739" } diff --git a/backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json b/backend/.sqlx/query-24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd.json similarity index 52% rename from backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json rename to backend/.sqlx/query-24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd.json index 58f693df33..7acbf9deec 100644 --- a/backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json +++ b/backend/.sqlx/query-24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id AS \"id!\" FROM v2_as_queue WHERE id = ANY($1) AND schedule_path IS NULL AND ($2::text[] IS NULL OR tag = ANY($2))", + "query": "SELECT j.id AS \"id!\" FROM v2_job j WHERE j.id = ANY($1) AND j.trigger_kind != 'schedule'::job_trigger_kind AND ($2::text[] IS NULL OR j.tag = ANY($2))", "describe": { "columns": [ { @@ -16,8 +16,8 @@ ] }, "nullable": [ - true + false ] }, - "hash": "acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359" + "hash": "24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd" } diff --git a/backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json b/backend/.sqlx/query-25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364.json similarity index 50% rename from backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json rename to backend/.sqlx/query-25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364.json index 804f35777e..7c07d169b8 100644 --- a/backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json +++ b/backend/.sqlx/query-25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT flow_status->'user_states'->$1\n FROM v2_as_queue\n WHERE id = $2 AND workspace_id = $3\n ", + "query": "\n SELECT COALESCE(s.flow_status, s.workflow_as_code_status)->'user_states'->$1\n FROM v2_job_queue q LEFT JOIN v2_job_status s USING (id)\n WHERE q.id = $2 AND q.workspace_id = $3\n ", "describe": { "columns": [ { @@ -20,5 +20,5 @@ null ] }, - "hash": "f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0" + "hash": "25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364" } diff --git a/backend/.sqlx/query-2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a.json b/backend/.sqlx/query-2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a.json new file mode 100644 index 0000000000..afa445d679 --- /dev/null +++ b/backend/.sqlx/query-2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now()\n OR (q.suspend_until IS NOT NULL\n AND (q.suspend <= 0 OR q.suspend_until <= now())))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a" +} diff --git a/backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json b/backend/.sqlx/query-29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99.json similarity index 58% rename from backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json rename to backend/.sqlx/query-29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99.json index 08bb98cc06..11f612ccda 100644 --- a/backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json +++ b/backend/.sqlx/query-29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\"\n FROM v2_as_completed_job\n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "query": "SELECT j.created_by AS \"created_by!\", j.args as \"args: sqlx::types::Json>\"\n FROM v2_job j\n WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { @@ -22,9 +22,9 @@ ] }, "nullable": [ - true, + false, true ] }, - "hash": "2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9" + "hash": "29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99" } diff --git a/backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json b/backend/.sqlx/query-35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b.json similarity index 54% rename from backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json rename to backend/.sqlx/query-35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b.json index a76515fd7e..e3d94ad1cf 100644 --- a/backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json +++ b/backend/.sqlx/query-35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", + "query": "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_job j\n LEFT JOIN job_logs ON job_logs.job_id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { @@ -32,11 +32,11 @@ ] }, "nullable": [ - true, + false, null, false, true ] }, - "hash": "ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9" + "hash": "35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b" } diff --git a/backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json b/backend/.sqlx/query-371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b.json similarity index 64% rename from backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json rename to backend/.sqlx/query-371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b.json index e9705c23a6..30550d5011 100644 --- a/backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json +++ b/backend/.sqlx/query-371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT script_path FROM v2_as_completed_job WHERE id = $1", + "query": "SELECT runnable_path as script_path FROM v2_job WHERE id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ true ] }, - "hash": "280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866" + "hash": "371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b" } diff --git a/backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json b/backend/.sqlx/query-37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950.json similarity index 55% rename from backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json rename to backend/.sqlx/query-37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950.json index ee5e15118d..c26f747c98 100644 --- a/backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json +++ b/backend/.sqlx/query-37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_as_queue\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id\n WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", + "query": "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_job j\n LEFT JOIN job_logs ON job_logs.job_id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { @@ -32,11 +32,11 @@ ] }, "nullable": [ - true, + false, null, null, true ] }, - "hash": "2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f" + "hash": "37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950" } diff --git a/backend/.sqlx/query-4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73.json b/backend/.sqlx/query-4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73.json new file mode 100644 index 0000000000..c1fd3462f7 --- /dev/null +++ b/backend/.sqlx/query-4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT j.tag as \"tag!\", COUNT(*) as \"count!\"\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE c.started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR j.workspace_id = $2)\n GROUP BY j.tag\n ORDER BY \"count!\" DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73" +} diff --git a/backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json b/backend/.sqlx/query-48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3.json similarity index 73% rename from backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json rename to backend/.sqlx/query-48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3.json index 45a36b4fe3..61c93e6573 100644 --- a/backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json +++ b/backend/.sqlx/query-48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n script_path, script_hash AS \"script_hash: ScriptHash\",\n job_kind AS \"job_kind!: JobKind\",\n flow_status AS \"flow_status: Json>\",\n raw_flow AS \"raw_flow: Json>\"\n FROM v2_as_completed_job WHERE id = $1 and workspace_id = $2", + "query": "SELECT\n j.runnable_path as script_path, j.runnable_id AS \"script_hash: ScriptHash\",\n j.kind AS \"job_kind!: JobKind\",\n COALESCE(c.flow_status, c.workflow_as_code_status) AS \"flow_status: Json>\",\n j.raw_flow AS \"raw_flow: Json>\"\n FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1 and j.workspace_id = $2", "describe": { "columns": [ { @@ -65,10 +65,10 @@ "nullable": [ true, true, - true, - true, + false, + null, true ] }, - "hash": "7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69" + "hash": "48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3" } diff --git a/backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json b/backend/.sqlx/query-49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e.json similarity index 65% rename from backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json rename to backend/.sqlx/query-49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e.json index 07ae8f17c5..d57628b1aa 100644 --- a/backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json +++ b/backend/.sqlx/query-49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n script_path, args AS \"args: sqlx::types::Json>>\",\n tag AS \"tag!\", priority\n FROM v2_as_completed_job\n WHERE id = $1 and workspace_id = $2", + "query": "SELECT\n j.runnable_path as script_path, j.args AS \"args: sqlx::types::Json>>\",\n j.tag AS \"tag!\", j.priority\n FROM v2_job j\n WHERE j.id = $1 and j.workspace_id = $2", "describe": { "columns": [ { @@ -33,9 +33,9 @@ "nullable": [ true, true, - true, + false, true ] }, - "hash": "2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f" + "hash": "49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e" } diff --git a/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json b/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json deleted file mode 100644 index b2c0b4702c..0000000000 --- a/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM v2_as_queue WHERE canceled = false AND (scheduled_for <= now()\n OR (suspend_until IS NOT NULL\n AND ( suspend <= 0\n OR suspend_until <= now())))", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb" -} diff --git a/backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json b/backend/.sqlx/query-57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825.json similarity index 50% rename from backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json rename to backend/.sqlx/query-57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825.json index 0a9e91b206..1bb262acbf 100644 --- a/backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json +++ b/backend/.sqlx/query-57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now() AND ($3::text[] IS NULL OR tag = ANY($3))", + "query": "SELECT coalesce(COUNT(*) FILTER(WHERE q.suspend = 0 AND q.running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE q.suspend > 0), 0) as \"suspended!\" FROM v2_job_queue q JOIN v2_job j USING (id) WHERE (j.workspace_id = $1 OR $2) AND q.scheduled_for <= now() AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { @@ -26,5 +26,5 @@ null ] }, - "hash": "0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd" + "hash": "57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825" } diff --git a/backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json b/backend/.sqlx/query-6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76.json similarity index 57% rename from backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json rename to backend/.sqlx/query-6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76.json index 3422605ef1..b2e0c68ba5 100644 --- a/backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json +++ b/backend/.sqlx/query-6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id AS \"id!\", flow_status, suspend AS \"suspend!\", script_path\n FROM v2_as_queue\n WHERE id = $1\n ", + "query": "\n SELECT j.id AS \"id!\", COALESCE(s.flow_status, s.workflow_as_code_status) as flow_status, q.suspend AS \"suspend!\", j.runnable_path as script_path\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE j.id = $1\n ", "describe": { "columns": [ { @@ -30,11 +30,11 @@ ] }, "nullable": [ - true, - true, - true, + false, + null, + false, true ] }, - "hash": "d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45" + "hash": "6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76" } diff --git a/backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json b/backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json new file mode 100644 index 0000000000..7cfd1e5b64 --- /dev/null +++ b/backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n j.id AS \"id!\", j.workspace_id AS \"workspace_id!\", j.parent_job, j.flow_step_id IS NOT NULL AS \"is_flow_step?\",\n COALESCE(s.flow_status, s.workflow_as_code_status) AS \"flow_status: Box\", r.ping AS last_ping, j.same_worker AS \"same_worker?\"\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now()\n AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode')\n AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval\n AND q.canceled_by IS NULL\n \n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "is_flow_step?", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "flow_status: Box", + "type_info": "Jsonb" + }, + { + "ordinal": 5, + "name": "last_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "same_worker?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + true, + null, + null, + true, + false + ] + }, + "hash": "70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b" +} diff --git a/backend/.sqlx/query-72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43.json b/backend/.sqlx/query-72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43.json deleted file mode 100644 index 2ffdf141b1..0000000000 --- a/backend/.sqlx/query-72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n id AS \"id!\", workspace_id AS \"workspace_id!\", parent_job, is_flow_step,\n flow_status AS \"flow_status: Box\", last_ping, same_worker\n FROM v2_as_queue\n WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now()\n AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode')\n AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval\n AND canceled = false\n \n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "workspace_id!", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "parent_job", - "type_info": "Uuid" - }, - { - "ordinal": 3, - "name": "is_flow_step", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "flow_status: Box", - "type_info": "Jsonb" - }, - { - "ordinal": 5, - "name": "last_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "same_worker", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43" -} diff --git a/backend/.sqlx/query-89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17.json b/backend/.sqlx/query-89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17.json deleted file mode 100644 index 25dd18003c..0000000000 --- a/backend/.sqlx/query-89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT usr.email, usage.executions\n FROM usr\n , LATERAL (\n SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_as_completed_job\n WHERE workspace_id = $1\n AND job_kind NOT IN ('flow', 'flowpreview', 'flownode')\n AND email = usr.email\n AND now() - '1 week'::interval < created_at\n ) usage\n WHERE workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "executions", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17" -} diff --git a/backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json b/backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json deleted file mode 100644 index 61598c0b2d..0000000000 --- a/backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT tag as \"tag!\", COUNT(*) as \"count!\"\n FROM v2_as_completed_job\n WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)\n GROUP BY tag\n ORDER BY \"count!\" DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "count!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Float8", - "Text" - ] - }, - "nullable": [ - true, - null - ] - }, - "hash": "90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71" -} diff --git a/backend/.sqlx/query-a001b4254e0f1ba8a87776e32f9506dce6eb4af3fb32e6a4c6e76084873a539a.json b/backend/.sqlx/query-a001b4254e0f1ba8a87776e32f9506dce6eb4af3fb32e6a4c6e76084873a539a.json new file mode 100644 index 0000000000..54ab7e320e --- /dev/null +++ b/backend/.sqlx/query-a001b4254e0f1ba8a87776e32f9506dce6eb4af3fb32e6a4c6e76084873a539a.json @@ -0,0 +1,216 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n j.id, j.workspace_id, j.runnable_id AS \"runnable_id!: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as, \n j.created_by, j.script_lang AS \"script_lang!: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind!: JobTriggerKind\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl\n FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id\n WHERE j.id = $1 AND j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "runnable_id!: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 6, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 7, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent" + ] + } + } + } + }, + { + "ordinal": 9, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "script_lang!: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 12, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "trigger_kind!: JobTriggerKind", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 17, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 18, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 19, + "name": "cache_ttl", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + true, + false, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + true + ] + }, + "hash": "a001b4254e0f1ba8a87776e32f9506dce6eb4af3fb32e6a4c6e76084873a539a" +} diff --git a/backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json b/backend/.sqlx/query-a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe.json similarity index 55% rename from backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json rename to backend/.sqlx/query-a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe.json index 1afb036030..d66a3aa9f0 100644 --- a/backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json +++ b/backend/.sqlx/query-a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) FROM v2_as_queue WHERE running = true AND email = $1", + "query": "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE q.running = true AND j.permissioned_as_email = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a" + "hash": "a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe" } diff --git a/backend/.sqlx/query-a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d.json b/backend/.sqlx/query-a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d.json new file mode 100644 index 0000000000..af4ada67f2 --- /dev/null +++ b/backend/.sqlx/query-a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $2\n AND (j.kind = 'appscript' OR j.kind = 'preview')\n AND j.created_by = 'anonymous'\n AND c.started_at > now() - interval '3 hours'\n AND j.runnable_path LIKE $3 || '/%'\n AND c.result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d" +} diff --git a/backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json b/backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json new file mode 100644 index 0000000000..73fb2e98d8 --- /dev/null +++ b/backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json @@ -0,0 +1,239 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, q.workspace_id, j.runnable_id as \"runnable_id: ScriptHash\", scheduled_for, parent_job, flow_innermost_root_job, runnable_path, kind as \"kind: JobKind\", started_at, permissioned_as, created_by, script_lang as \"script_lang: ScriptLang\", \n permissioned_as_email, flow_step_id, trigger_kind as \"trigger_kind: JobTriggerKind\", trigger, q.priority, concurrent_limit, q.tag, cache_ttl, r.ping as last_ping, worker, memory_peak, running\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE j.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "runnable_id: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 5, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 6, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "kind: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent" + ] + } + } + } + }, + { + "ordinal": 8, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 12, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "trigger_kind: JobTriggerKind", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 17, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 18, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 19, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 20, + "name": "last_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 21, + "name": "worker", + "type_info": "Varchar" + }, + { + "ordinal": 22, + "name": "memory_peak", + "type_info": "Int4" + }, + { + "ordinal": 23, + "name": "running", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + false, + true, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd" +} diff --git a/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json b/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json deleted file mode 100644 index d87e680abe..0000000000 --- a/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS (\n SELECT 1 FROM v2_as_completed_job\n WHERE workspace_id = $2\n AND (job_kind = 'appscript' OR job_kind = 'preview')\n AND created_by = 'anonymous'\n AND started_at > now() - interval '3 hours'\n AND script_path LIKE $3 || '/%'\n AND result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb\n )", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b" -} diff --git a/backend/.sqlx/query-edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd.json b/backend/.sqlx/query-edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd.json new file mode 100644 index 0000000000..f67e52ab6b --- /dev/null +++ b/backend/.sqlx/query-edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "executions", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd" +} diff --git a/backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json b/backend/.sqlx/query-fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e.json similarity index 52% rename from backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json rename to backend/.sqlx/query-fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e.json index 9cda5cc93e..29cedfbc1c 100644 --- a/backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json +++ b/backend/.sqlx/query-fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH job_info AS (\n -- Query for Teams (running jobs)\n SELECT\n parent.job_kind AS \"job_kind!: JobKind\",\n parent.script_hash AS \"script_hash: ScriptHash\",\n parent.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n child.parent_job AS \"parent_job: Uuid\",\n parent.created_at AS \"created_at!: chrono::NaiveDateTime\",\n parent.created_by AS \"created_by!\",\n parent.script_path,\n parent.args AS \"args: sqlx::types::Json>\"\n FROM v2_as_queue child\n JOIN v2_as_queue parent ON parent.id = child.parent_job\n WHERE child.id = $1 AND child.workspace_id = $2\n UNION ALL\n -- Query for Slack (completed jobs)\n SELECT\n v2_as_queue.job_kind AS \"job_kind!: JobKind\",\n v2_as_queue.script_hash AS \"script_hash: ScriptHash\",\n v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n v2_as_completed_job.parent_job AS \"parent_job: Uuid\",\n v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\",\n v2_as_completed_job.created_by AS \"created_by!\",\n v2_as_queue.script_path,\n v2_as_queue.args AS \"args: sqlx::types::Json>\"\n FROM v2_as_queue\n JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2\n )\n SELECT * FROM job_info LIMIT 1", + "query": "WITH job_info AS (\n -- Query for Teams (running jobs)\n SELECT\n parent_j.kind AS \"job_kind!: JobKind\",\n parent_j.runnable_id AS \"script_hash: ScriptHash\",\n parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n child_j.parent_job AS \"parent_job: Uuid\",\n parent_j.created_at AS \"created_at!: chrono::NaiveDateTime\",\n parent_j.created_by AS \"created_by!\",\n parent_j.runnable_path as script_path,\n parent_j.args AS \"args: sqlx::types::Json>\"\n FROM v2_job_queue child_q JOIN v2_job child_j USING (id)\n JOIN v2_job parent_j ON parent_j.id = child_j.parent_job\n WHERE child_j.id = $1 AND child_j.workspace_id = $2\n UNION ALL\n -- Query for Slack (completed jobs)\n SELECT\n parent_j.kind AS \"job_kind!: JobKind\",\n parent_j.runnable_id AS \"script_hash: ScriptHash\",\n parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n completed_j.parent_job AS \"parent_job: Uuid\",\n completed_j.created_at AS \"created_at!: chrono::NaiveDateTime\",\n completed_j.created_by AS \"created_by!\",\n parent_j.runnable_path as script_path,\n parent_j.args AS \"args: sqlx::types::Json>\"\n FROM v2_job_completed completed_c JOIN v2_job completed_j USING (id)\n JOIN v2_job parent_j ON parent_j.id = completed_j.parent_job\n WHERE completed_j.id = $1 AND completed_j.workspace_id = $2\n )\n SELECT * FROM job_info LIMIT 1", "describe": { "columns": [ { @@ -88,5 +88,5 @@ null ] }, - "hash": "3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e" + "hash": "fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e" } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index e310f0e3bf..b0818c96cd 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -59,7 +59,6 @@ use windmill_common::{ SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, indexer::load_indexer_config, - jobs::QueuedJob, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, @@ -80,7 +79,7 @@ use windmill_common::{ OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, }; use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING}; -use windmill_queue::{cancel_job, SameWorkerPayload}; +use windmill_queue::{SameWorkerPayload, cancel_job, get_queued_job_v2}; use windmill_worker::{ handle_job_error, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, @@ -2047,7 +2046,7 @@ async fn cancel_stale_job( const RESTART_LIMIT: i32 = 3; -async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker_name: &str) { +async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_name: &str) { let mut zombie_jobs_uuid_restart_limit_reached = vec![]; if *RESTART_ZOMBIE_JOBS { @@ -2220,24 +2219,18 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker ); } - let jobs = sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status FROM v2_as_queue WHERE id = ANY($1)", - ) - .bind(&timeouts[..]) - .fetch_all(db) - .await - .map_err(|e| tracing::error!("Error fetching same worker jobs: {:?}", e)) - .unwrap_or_default(); - jobs + timeouts }; let non_restartable_jobs = if *RESTART_ZOMBIE_JOBS { vec![] } else { - sqlx::query_as::<_, QueuedJob>("SELECT *, null as workflow_as_code_status FROM v2_as_queue WHERE last_ping < now() - ($1 || ' seconds')::interval - AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND same_worker = false") - .bind(ZOMBIE_JOB_TIMEOUT.as_str()) + sqlx::query_scalar!("SELECT j.id + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE r.ping < now() - ($1 || ' seconds')::interval + AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false", + ZOMBIE_JOB_TIMEOUT.as_str()) .fetch_all(db) .await .ok() @@ -2260,14 +2253,6 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker } } - let zombie_jobs_restart_limit_reached = sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status FROM v2_as_queue WHERE id = ANY($1)", - ) - .bind(&zombie_jobs_uuid_restart_limit_reached[..]) - .fetch_all(db) - .await - .ok() - .unwrap_or_else(|| vec![]); let timeouts = non_restartable_jobs .into_iter() @@ -2278,7 +2263,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker .map(|x| (x, ErrorMessage::SameWorker)), ) .chain( - zombie_jobs_restart_limit_reached + zombie_jobs_uuid_restart_limit_reached .into_iter() .map(|x| (x, ErrorMessage::RestartLimit)), ) @@ -2289,7 +2274,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _); } - for (job, error_kind) in timeouts { + for (job_id, error_kind) in timeouts { // since the job is unrecoverable, the same worker queue should never be sent anything let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::(1); @@ -2298,6 +2283,12 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker let (send_result_never_used, _send_result_rx_never_used) = JobCompletedSender::new_never_used(); + let job = get_queued_job_v2(db, &job_id).await; + if let Err(e) = job { + tracing::error!("Error getting queued job: {:?}", e); + continue; + } + if let Some(job) = job.unwrap() { let label = if job.permissioned_as != format!("u/{}", job.created_by) && job.permissioned_as != job.created_by { @@ -2311,7 +2302,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker &job.permissioned_as, &label, *SCRIPT_TOKEN_EXPIRY, - &job.email, + &job.permissioned_as_email, &job.id, None, Some(format!("handle_zombie_jobs")), @@ -2326,32 +2317,32 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker None, ); - let last_ping = job.last_ping.clone(); let error_message = format!( - "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, reason: {:?})", - last_ping - .map(|x| x.to_string()) - .unwrap_or_else(|| "no ping".to_string()), + "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, reason: {:?}).\nThis likely means that the job died on worker {}, OOM are a common reason for worker crashes.\nCheck the workers around the time of the last ping and the exit code if any.", + job.last_ping.unwrap_or_default(), *ZOMBIE_JOB_TIMEOUT, - error_kind.to_string() + error_kind.to_string(), + job.worker.clone().unwrap_or_default(), ); + let memory_peak = job.memory_peak.unwrap_or(0); let _ = handle_job_error( db, &client, - &windmill_queue::MiniCompletedJob::from(windmill_queue::MiniPulledJob::from(&job)), - 0, + &windmill_queue::MiniCompletedJob::from(job), + memory_peak, None, error::Error::ExecutionErr(error_message), true, Some(&same_worker_tx_never_used), "", - worker_name, + node_name, send_result_never_used, #[cfg(feature = "benchmark")] &mut windmill_common::bench::BenchmarkIter::new(), ) .await; } + } } async fn cleanup_concurrency_counters_orphaned_keys(db: &DB) -> error::Result<()> { @@ -2457,13 +2448,13 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { let flows = sqlx::query!( r#" SELECT - id AS "id!", workspace_id AS "workspace_id!", parent_job, is_flow_step, - flow_status AS "flow_status: Box", last_ping, same_worker - FROM v2_as_queue - WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now() - AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode') - AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval - AND canceled = false + j.id AS "id!", j.workspace_id AS "workspace_id!", j.parent_job, j.flow_step_id IS NOT NULL AS "is_flow_step?", + COALESCE(s.flow_status, s.workflow_as_code_status) AS "flow_status: Box", r.ping AS last_ping, j.same_worker AS "same_worker?" + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now() + AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode') + AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval + AND q.canceled_by IS NULL "#, FLOW_ZOMBIE_TRANSITION_TIMEOUT.as_str() diff --git a/backend/tests/common/mod.rs b/backend/tests/common/mod.rs index 475a8540b7..40a9980a96 100644 --- a/backend/tests/common/mod.rs +++ b/backend/tests/common/mod.rs @@ -394,7 +394,17 @@ pub async fn listen_for_uuid_on( pub async fn completed_job(uuid: Uuid, db: &Pool) -> CompletedJob { sqlx::query_as::<_, CompletedJob>( - "SELECT *, result->'wm_labels' as labels FROM v2_as_completed_job WHERE id = $1", + "SELECT j.id, j.workspace_id, j.parent_job, j.created_by, j.created_at, c.duration_ms, + c.status = 'success' OR c.status = 'skipped' AS success, j.runnable_id AS script_hash, j.runnable_path AS script_path, + j.args, c.result, FALSE AS deleted, j.raw_code, c.status = 'canceled' AS canceled, + c.canceled_by, c.canceled_reason, j.kind AS job_kind, + CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END AS schedule_path, + j.permissioned_as, COALESCE(c.flow_status, c.workflow_as_code_status) AS flow_status, j.raw_flow, + j.flow_step_id IS NOT NULL AS is_flow_step, j.script_lang AS language, c.started_at, + c.status = 'skipped' AS is_skipped, j.raw_lock, j.permissioned_as_email AS email, j.visible_to_owner, + c.memory_peak AS mem_peak, j.tag, j.priority, NULL::TEXT AS logs, c.result_columns, + j.script_entrypoint_override, j.preprocessed, c.result->'wm_labels' as labels + FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1", ) .bind(uuid) .fetch_one(db) diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 1129846bc9..72b6aeea52 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -2395,7 +2395,7 @@ async fn test_script_schedule_handlers(db: Pool) -> anyhow::Result<()> let uuid = uuid.unwrap().unwrap(); let completed_job = sqlx::query!( - "SELECT script_path FROM v2_as_completed_job WHERE id = $1", + "SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid ) .fetch_one(&db2) @@ -2466,7 +2466,7 @@ async fn test_script_schedule_handlers(db: Pool) -> anyhow::Result<()> let uuid = uuid.unwrap().unwrap(); let completed_job = - sqlx::query!("SELECT script_path FROM v2_as_completed_job WHERE id = $1", uuid) + sqlx::query!("SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid) .fetch_one(&db2) .await .unwrap(); @@ -2553,7 +2553,7 @@ async fn test_flow_schedule_handlers(db: Pool) -> anyhow::Result<()> { let uuid = uuid.unwrap().unwrap(); let completed_job = sqlx::query!( - "SELECT script_path FROM v2_as_completed_job WHERE id = $1", + "SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid ) .fetch_one(&db2) @@ -2625,7 +2625,7 @@ async fn test_flow_schedule_handlers(db: Pool) -> anyhow::Result<()> { let uuid = uuid.unwrap().unwrap(); let completed_job = - sqlx::query!("SELECT script_path FROM v2_as_completed_job WHERE id = $1", uuid) + sqlx::query!("SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid) .fetch_one(&db2) .await .unwrap(); diff --git a/backend/windmill-api/src/approvals.rs b/backend/windmill-api/src/approvals.rs index e03b587f10..1296000481 100644 --- a/backend/windmill-api/src/approvals.rs +++ b/backend/windmill-api/src/approvals.rs @@ -204,31 +204,31 @@ pub async fn get_approval_form_details( "WITH job_info AS ( -- Query for Teams (running jobs) SELECT - parent.job_kind AS \"job_kind!: JobKind\", - parent.script_hash AS \"script_hash: ScriptHash\", - parent.raw_flow AS \"raw_flow: sqlx::types::Json>\", - child.parent_job AS \"parent_job: Uuid\", - parent.created_at AS \"created_at!: chrono::NaiveDateTime\", - parent.created_by AS \"created_by!\", - parent.script_path, - parent.args AS \"args: sqlx::types::Json>\" - FROM v2_as_queue child - JOIN v2_as_queue parent ON parent.id = child.parent_job - WHERE child.id = $1 AND child.workspace_id = $2 + parent_j.kind AS \"job_kind!: JobKind\", + parent_j.runnable_id AS \"script_hash: ScriptHash\", + parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\", + child_j.parent_job AS \"parent_job: Uuid\", + parent_j.created_at AS \"created_at!: chrono::NaiveDateTime\", + parent_j.created_by AS \"created_by!\", + parent_j.runnable_path as script_path, + parent_j.args AS \"args: sqlx::types::Json>\" + FROM v2_job_queue child_q JOIN v2_job child_j USING (id) + JOIN v2_job parent_j ON parent_j.id = child_j.parent_job + WHERE child_j.id = $1 AND child_j.workspace_id = $2 UNION ALL -- Query for Slack (completed jobs) SELECT - v2_as_queue.job_kind AS \"job_kind!: JobKind\", - v2_as_queue.script_hash AS \"script_hash: ScriptHash\", - v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\", - v2_as_completed_job.parent_job AS \"parent_job: Uuid\", - v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\", - v2_as_completed_job.created_by AS \"created_by!\", - v2_as_queue.script_path, - v2_as_queue.args AS \"args: sqlx::types::Json>\" - FROM v2_as_queue - JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id - WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 + parent_j.kind AS \"job_kind!: JobKind\", + parent_j.runnable_id AS \"script_hash: ScriptHash\", + parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\", + completed_j.parent_job AS \"parent_job: Uuid\", + completed_j.created_at AS \"created_at!: chrono::NaiveDateTime\", + completed_j.created_by AS \"created_by!\", + parent_j.runnable_path as script_path, + parent_j.args AS \"args: sqlx::types::Json>\" + FROM v2_job_completed completed_c JOIN v2_job completed_j USING (id) + JOIN v2_job parent_j ON parent_j.id = completed_j.parent_job + WHERE completed_j.id = $1 AND completed_j.workspace_id = $2 ) SELECT * FROM job_info LIMIT 1", job_id, diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 1cd0750919..b669f5e9a9 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -2530,13 +2530,13 @@ async fn check_if_allowed_to_access_s3_file_from_app( || { sqlx::query_scalar!( r#"SELECT EXISTS ( - SELECT 1 FROM v2_as_completed_job - WHERE workspace_id = $2 - AND (job_kind = 'appscript' OR job_kind = 'preview') - AND created_by = 'anonymous' - AND started_at > now() - interval '3 hours' - AND script_path LIKE $3 || '/%' - AND result @> ('{"s3":"' || $1 || '"}')::jsonb + SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE j.workspace_id = $2 + AND (j.kind = 'appscript' OR j.kind = 'preview') + AND j.created_by = 'anonymous' + AND c.started_at > now() - interval '3 hours' + AND j.runnable_path LIKE $3 || '/%' + AND c.result @> ('{"s3":"' || $1 || '"}')::jsonb )"#, file_query.s3, w_id, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 4c11515291..0293b4b625 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -841,101 +841,6 @@ macro_rules! get_job_query { } } -// CREATE OR REPLACE VIEW v2_as_queue AS -// SELECT -// j.id, -// j.workspace_id, -// j.parent_job, -// j.created_by, -// j.created_at, -// q.started_at, -// q.scheduled_for, -// q.running, -// j.runnable_id AS script_hash, -// j.runnable_path AS script_path, -// j.args, -// j.raw_code, -// q.canceled_by IS NOT NULL AS canceled, -// q.canceled_by, -// q.canceled_reason, -// r.ping AS last_ping, -// j.kind AS job_kind, -// CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END -// AS schedule_path, -// j.permissioned_as, -// COALESCE(s.flow_status, s.workflow_as_code_status) AS flow_status, -// j.raw_flow, -// j.flow_step_id IS NOT NULL AS is_flow_step, -// j.script_lang AS language, -// q.suspend, -// q.suspend_until, -// j.same_worker, -// j.raw_lock, -// j.pre_run_error, -// j.permissioned_as_email AS email, -// j.visible_to_owner, -// r.memory_peak AS mem_peak, -// j.flow_innermost_root_job AS root_job, -// s.flow_leaf_jobs AS leaf_jobs, -// j.tag, -// j.concurrent_limit, -// j.concurrency_time_window_s, -// j.timeout, -// j.flow_step_id, -// j.cache_ttl, -// j.priority, -// NULL::TEXT AS logs, -// j.script_entrypoint_override, -// j.preprocessed -// FROM v2_job_queue q -// JOIN v2_job j USING (id) -// LEFT JOIN v2_job_runtime r USING (id) -// LEFT JOIN v2_job_status s USING (id) -// ; - -// -- Add up migration script here -// CREATE OR REPLACE VIEW v2_as_completed_job AS -// SELECT -// j.id, -// j.workspace_id, -// j.parent_job, -// j.created_by, -// j.created_at, -// c.duration_ms, -// c.status = 'success' OR c.status = 'skipped' AS success, -// j.runnable_id AS script_hash, -// j.runnable_path AS script_path, -// j.args, -// c.result, -// FALSE AS deleted, -// j.raw_code, -// c.status = 'canceled' AS canceled, -// c.canceled_by, -// c.canceled_reason, -// j.kind AS job_kind, -// CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END -// AS schedule_path, -// j.permissioned_as, -// COALESCE(c.flow_status, c.workflow_as_code_status) AS flow_status, -// j.raw_flow, -// j.flow_step_id IS NOT NULL AS is_flow_step, -// j.script_lang AS language, -// c.started_at, -// c.status = 'skipped' AS is_skipped, -// j.raw_lock, -// j.permissioned_as_email AS email, -// j.visible_to_owner, -// c.memory_peak AS mem_peak, -// j.tag, -// j.priority, -// NULL::TEXT AS logs, -// c.result_columns, -// j.script_entrypoint_override, -// j.preprocessed -// FROM v2_job_completed c -// JOIN v2_job j USING (id) -// ; - #[derive(Copy, Clone)] struct GetQuery<'a> { with_logs: bool, @@ -1506,10 +1411,10 @@ async fn get_job_logs( .flatten(); let record = sqlx::query!( - "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index - FROM v2_as_completed_job - LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id - WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", + "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index + FROM v2_job j + LEFT JOIN job_logs ON job_logs.job_id = j.id + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", id, w_id, tags.as_ref().map(|v| v.as_slice()) @@ -1554,10 +1459,10 @@ async fn get_job_logs( Ok(content_plain(Body::from(logs))) } else { let text = sqlx::query!( - "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index - FROM v2_as_queue - LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id - WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", + "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index + FROM v2_job j + LEFT JOIN job_logs ON job_logs.job_id = j.id + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", id, w_id, tags.as_ref().map(|v| v.as_slice()) @@ -1613,9 +1518,9 @@ async fn get_args( .map(|authed| get_scope_tags(authed)) .flatten(); let record = sqlx::query!( - "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\" - FROM v2_as_completed_job - WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "SELECT j.created_by AS \"created_by!\", j.args as \"args: sqlx::types::Json>\" + FROM v2_job j + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", id, &w_id, tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, @@ -2170,7 +2075,7 @@ async fn cancel_selection( let mut tx = user_db.begin(&authed).await?; let tags = get_scope_tags(&authed).map(|v| v.iter().map(|s| s.to_string()).collect_vec()); let jobs_to_cancel = sqlx::query_scalar!( - "SELECT id AS \"id!\" FROM v2_as_queue WHERE id = ANY($1) AND schedule_path IS NULL AND ($2::text[] IS NULL OR tag = ANY($2))", + "SELECT j.id AS \"id!\" FROM v2_job j WHERE j.id = ANY($1) AND j.trigger_kind != 'schedule'::job_trigger_kind AND ($2::text[] IS NULL OR j.tag = ANY($2))", &jobs, tags.as_ref().map(|v| v.as_slice()) ) @@ -2274,7 +2179,7 @@ async fn count_queue_jobs( Ok(Json( sqlx::query_as!( QueueStats, - "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now() AND ($3::text[] IS NULL OR tag = ANY($3))", + "SELECT coalesce(COUNT(*) FILTER(WHERE q.suspend = 0 AND q.running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE q.suspend > 0), 0) as \"suspended!\" FROM v2_job_queue q JOIN v2_job j USING (id) WHERE (j.workspace_id = $1 OR $2) AND q.scheduled_for <= now() AND ($3::text[] IS NULL OR j.tag = ANY($3))", w_id, w_id == "admins" && cq.all_workspaces.unwrap_or(false), tags.as_ref().map(|v| v.as_slice()) @@ -2717,9 +2622,9 @@ async fn get_suspended_flow_info<'c>( let flow = sqlx::query_as!( FlowInfo, r#" - SELECT id AS "id!", flow_status, suspend AS "suspend!", script_path - FROM v2_as_queue - WHERE id = $1 + SELECT j.id AS "id!", COALESCE(s.flow_status, s.workflow_as_code_status) as flow_status, q.suspend AS "suspend!", j.runnable_path as script_path + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE j.id = $1 "#, job_id, ) @@ -2933,9 +2838,9 @@ pub async fn get_flow_user_state( let mut tx = user_db.begin(&authed).await?; let r = sqlx::query_scalar!( r#" - SELECT flow_status->'user_states'->$1 - FROM v2_as_queue - WHERE id = $2 AND workspace_id = $3 + SELECT COALESCE(s.flow_status, s.workflow_as_code_status)->'user_states'->$1 + FROM v2_job_queue q LEFT JOIN v2_job_status s USING (id) + WHERE q.id = $2 AND q.workspace_id = $3 "#, key, job_id, @@ -4166,10 +4071,10 @@ pub async fn restart_flow( let mut tx = user_db.clone().begin(&authed).await?; let completed_job = sqlx::query!( "SELECT - script_path, args AS \"args: sqlx::types::Json>>\", - tag AS \"tag!\", priority - FROM v2_as_completed_job - WHERE id = $1 and workspace_id = $2", + j.runnable_path as script_path, j.args AS \"args: sqlx::types::Json>>\", + j.tag AS \"tag!\", j.priority + FROM v2_job j + WHERE j.id = $1 and j.workspace_id = $2", job_id, &w_id, ) @@ -4819,10 +4724,9 @@ pub async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<() pub async fn check_queue_too_long(db: &DB, queue_limit: Option) -> error::Result<()> { if let Some(limit) = queue_limit { let count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM v2_as_queue WHERE canceled = false AND (scheduled_for <= now() - OR (suspend_until IS NOT NULL - AND ( suspend <= 0 - OR suspend_until <= now())))", + "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now() + OR (q.suspend_until IS NOT NULL + AND (q.suspend <= 0 OR q.suspend_until <= now())))", ) .fetch_one(db) .await? @@ -7963,10 +7867,10 @@ async fn count_by_tag( let counts = sqlx::query_as!( TagCount, r#" - SELECT tag as "tag!", COUNT(*) as "count!" - FROM v2_as_completed_job - WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2) - GROUP BY tag + SELECT j.tag as "tag!", COUNT(*) as "count!" + FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE c.started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR j.workspace_id = $2) + GROUP BY j.tag ORDER BY "count!" DESC "#, horizon as f64, diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 165316ba56..9021d6fe8e 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -494,14 +494,13 @@ async fn list_user_usage( UserWithUsage, " SELECT usr.email, usage.executions - FROM usr - , LATERAL ( - SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions - FROM v2_as_completed_job - WHERE workspace_id = $1 - AND job_kind NOT IN ('flow', 'flowpreview', 'flownode') - AND email = usr.email - AND now() - '1 week'::interval < created_at + FROM usr, LATERAL ( + SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions + FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE j.workspace_id = $1 + AND j.kind NOT IN ('flow', 'flowpreview', 'flownode') + AND j.permissioned_as_email = usr.email + AND now() - '1 week'::interval < j.created_at ) usage WHERE workspace_id = $1 ", diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 4311972923..f179216d24 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -6,6 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ +use std::future::Future; use std::{collections::HashMap, sync::Arc, vec}; use anyhow::Context; @@ -154,18 +155,19 @@ pub struct JobCompleted { pub async fn cancel_single_job<'c>( username: &str, reason: Option, - job_running: Arc, + job_running: QueuedJobV2, w_id: &str, mut tx: Transaction<'c, Postgres>, db: &Pool, force_cancel: bool, ) -> error::Result<(Transaction<'c, Postgres>, Option)> { + + let id = job_running.id; if force_cancel || (job_running.parent_job.is_none() && !job_running.running) { let username = username.to_string(); let w_id = w_id.to_string(); let db = db.clone(); tracing::info!("cancelling job {:?}", job_running.id); - let job_running = job_running.clone(); tokio::task::spawn(async move { let reason: String = reason .clone() @@ -179,10 +181,11 @@ pub async fn cancel_single_job<'c>( &Connection::from(db.clone()), ) .await; + let memory_peak = job_running.memory_peak.unwrap_or(0); let add_job = add_completed_job_error( &db, - &MiniCompletedJob::from(MiniPulledJob::from(&job_running)), - job_running.mem_peak.unwrap_or(0), + &MiniCompletedJob::from(job_running), + memory_peak, Some(CanceledBy { username: Some(username.to_string()), reason: Some(reason) }), e, "server", @@ -210,7 +213,7 @@ pub async fn cancel_single_job<'c>( } } - Ok((tx, Some(job_running.id))) + Ok((tx, Some(id))) } pub async fn cancel_job<'c>( @@ -224,26 +227,35 @@ pub async fn cancel_job<'c>( require_anonymous: bool, ) -> error::Result<(Transaction<'c, Postgres>, Option)> { //TODO fetch mini completed job instead of QueuedJob - let job = get_queued_job_tx(id, &w_id, &mut tx).await?; + let job = get_queued_job_v2(&mut *tx, &id).await?; + if job.is_none() { return Ok((tx, None)); } - if require_anonymous && job.as_ref().unwrap().created_by != "anonymous" { + let mut job = job.unwrap(); + + if require_anonymous && job.created_by != "anonymous" { return Err(Error::BadRequest( "You are not logged in and this job was not created by an anonymous user like you so you cannot cancel it".to_string(), )); } - let mut job = job.unwrap(); + + if job.workspace_id != w_id { + return Err(Error::BadRequest( + "You are not authorized to cancel this job belonging to another workspace".to_string(), + )); + } + if force_cancel { // if force canceling a flow step, make sure we force cancel from the highest parent loop { if job.parent_job.is_none() { break; } - match get_queued_job_tx(job.parent_job.unwrap(), &w_id, &mut tx).await? { + match get_queued_job_v2(&mut *tx, &job.parent_job.unwrap()).await? { Some(j) => { job = j; } @@ -253,7 +265,7 @@ pub async fn cancel_job<'c>( } // prevent cancelling a future tick of a schedule - if let Some(schedule_path) = job.schedule_path.as_ref() { + if let Some(schedule_path) = job.schedule_path().as_ref() { let now = now_from_db(&mut *tx).await?; if job.scheduled_for > now { return Err(Error::BadRequest( @@ -266,7 +278,7 @@ pub async fn cancel_job<'c>( } } - let job = Arc::new(job); + let job = job; // get all children using recursive CTE let mut jobs_to_cancel = sqlx::query!( @@ -308,7 +320,7 @@ ORDER BY depth, id let (ntx, _) = cancel_single_job( username, reason.clone(), - job.clone(), + job, w_id, tx, db, @@ -335,13 +347,13 @@ ORDER BY depth, id } } for job_id in jobs_to_cancel { - let job = get_queued_job_tx(job_id, &w_id, &mut tx).await?; + let job = get_queued_job_v2(&mut *tx, &job_id).await?; if let Some(job) = job { let (ntx, _) = cancel_single_job( username, reason.clone(), - Arc::new(job), + job, w_id, tx, db, @@ -554,7 +566,7 @@ async fn cancel_persistent_script_jobs_internal<'c>( // we could have retrieved the job IDs in the first query where we retrieve the hashes, but just in case a job was inserted in the queue right in-between the two above query, we re-do the fetch here let jobs_to_cancel = sqlx::query_scalar::<_, Uuid>( - "SELECT id FROM v2_as_queue WHERE workspace_id = $1 AND script_path = $2 AND canceled = false", + "SELECT j.id FROM v2_job_queue q JOIN v2_job j USING (id) WHERE j.workspace_id = $1 AND j.runnable_path = $2 AND q.canceled_by IS NULL", ) .bind(w_id) .bind(script_path) @@ -1948,6 +1960,34 @@ pub struct MiniCompletedJob { pub cache_ttl: Option, } +impl From for MiniCompletedJob { + fn from(job: QueuedJobV2) -> Self { + MiniCompletedJob { + id: job.id, + workspace_id: job.workspace_id, + runnable_id: job.runnable_id, + scheduled_for: job.scheduled_for, + parent_job: job.parent_job, + flow_innermost_root_job: job.flow_innermost_root_job, + runnable_path: job.runnable_path, + kind: job.kind, + started_at: job.started_at, + permissioned_as: job.permissioned_as, + created_by: job.created_by, + script_lang: job.script_lang, + permissioned_as_email: job.permissioned_as_email, + flow_step_id: job.flow_step_id, + trigger_kind: job.trigger_kind, + trigger: job.trigger, + priority: job.priority, + concurrent_limit: job.concurrent_limit, + tag: job.tag, + cache_ttl: job.cache_ttl, + + } + } +} + impl From for MiniCompletedJob { fn from(job: MiniPulledJob) -> Self { MiniCompletedJob { @@ -2008,11 +2048,7 @@ impl MiniCompletedJob { self.flow_step_id.is_some() } pub fn schedule_path(&self) -> Option { - if self.trigger_kind.as_ref().is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) { - self.trigger.clone() - } else { - None - } + schedule_path(&self.trigger_kind, &self.trigger) } pub fn is_flow(&self) -> bool { @@ -2025,6 +2061,14 @@ impl MiniCompletedJob { } +fn schedule_path(trigger_kind: &Option, trigger: &Option) -> Option { + if trigger_kind.as_ref().is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) { + trigger.clone() + } else { + None + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] struct FlowStatusChatInputEnabled { chat_input_enabled: Option, @@ -2111,15 +2155,7 @@ impl MiniPulledJob { } pub fn schedule_path(&self) -> Option { - if self - .trigger_kind - .as_ref() - .is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) - { - self.trigger.clone() - } else { - None - } + schedule_path(&self.trigger_kind, &self.trigger) } pub async fn mark_as_started_if_step(&self, db: &DB) -> Result<(), Error> { @@ -2316,6 +2352,57 @@ pub async fn get_mini_pulled_job<'c>( Ok(job) } + +pub struct QueuedJobV2 { + pub id: Uuid, + pub workspace_id: String, + pub runnable_id: Option, + pub scheduled_for: chrono::DateTime, + pub parent_job: Option, + // pub root_job: Option, + pub flow_innermost_root_job: Option, + pub runnable_path: Option, + pub kind: JobKind, + pub started_at: Option>, + pub permissioned_as: String, + pub created_by: String, + pub script_lang: Option, + pub permissioned_as_email: String, + pub flow_step_id: Option, + pub trigger_kind: Option, + pub trigger: Option, + pub priority: Option, + pub concurrent_limit: Option, + pub tag: String, + pub cache_ttl: Option, + pub last_ping: Option>, + pub worker: Option, + pub memory_peak: Option, + pub running: bool, +} + +impl QueuedJobV2 { + pub fn schedule_path(&self) -> Option { + schedule_path(&self.trigger_kind, &self.trigger) + } +} + +pub async fn get_queued_job_v2<'c>( + e: impl PgExecutor<'c>, job_id: &Uuid) -> error::Result> { + let job = sqlx::query_as!( + QueuedJobV2, + "SELECT id, q.workspace_id, j.runnable_id as \"runnable_id: ScriptHash\", scheduled_for, parent_job, flow_innermost_root_job, runnable_path, kind as \"kind: JobKind\", started_at, permissioned_as, created_by, script_lang as \"script_lang: ScriptLang\", + permissioned_as_email, flow_step_id, trigger_kind as \"trigger_kind: JobTriggerKind\", trigger, q.priority, concurrent_limit, q.tag, cache_ttl, r.ping as last_ping, worker, memory_peak, running + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE j.id = $1", + job_id, + + ) + .fetch_optional(e) + .await?; + Ok(job) +} + #[derive(Serialize, Deserialize, Debug)] pub struct PulledJobResult { pub job: Option, @@ -3170,33 +3257,29 @@ pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result( - id: Uuid, - w_id: &str, - tx: &mut Transaction<'c, Postgres>, -) -> error::Result> { - sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status - FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", - ) - .bind(id) - .bind(w_id) - .fetch_optional(&mut **tx) - .await - .map_err(Into::into) +pub fn get_mini_completed_job< +'a, +'e, +A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a, +>(id: &'a Uuid, w_id: &'a str, db: A) -> impl Future>> + Send + 'a { + async move { + let mut conn = db.acquire().await?; + sqlx::query_as!( + MiniCompletedJob, + "SELECT + j.id, j.workspace_id, j.runnable_id AS \"runnable_id!: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as, + j.created_by, j.script_lang AS \"script_lang!: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind!: JobTriggerKind\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl + FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id + WHERE j.id = $1 AND j.workspace_id = $2", + id, + w_id + ) + .fetch_optional(&mut *conn) + .await + .map_err(Into::into) + } } -pub async fn get_queued_job(id: &Uuid, w_id: &str, db: &DB) -> error::Result> { - sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status - FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", - ) - .bind(id) - .bind(w_id) - .fetch_optional(db) - .await - .map_err(Into::into) -} pub enum PushIsolationLevel<'c> { IsolatedRoot(DB), @@ -3557,7 +3640,7 @@ pub async fn push<'c, 'd>( } let in_queue = sqlx::query_scalar!( - "SELECT COUNT(id) FROM v2_as_queue WHERE email = $1", + "SELECT COUNT(id) FROM v2_job WHERE permissioned_as_email = $1", email ) .fetch_one(_db) @@ -3571,7 +3654,7 @@ pub async fn push<'c, 'd>( } let concurrent_runs = sqlx::query_scalar!( - "SELECT COUNT(id) FROM v2_as_queue WHERE running = true AND email = $1", + "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE q.running = true AND j.permissioned_as_email = $1", email ) .fetch_one(_db) @@ -5155,11 +5238,11 @@ async fn restarted_flows_resolution( > { let row = sqlx::query!( "SELECT - script_path, script_hash AS \"script_hash: ScriptHash\", - job_kind AS \"job_kind!: JobKind\", - flow_status AS \"flow_status: Json>\", - raw_flow AS \"raw_flow: Json>\" - FROM v2_as_completed_job WHERE id = $1 and workspace_id = $2", + j.runnable_path as script_path, j.runnable_id AS \"script_hash: ScriptHash\", + j.kind AS \"job_kind!: JobKind\", + COALESCE(c.flow_status, c.workflow_as_code_status) AS \"flow_status: Json>\", + j.raw_flow AS \"raw_flow: Json>\" + FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1 and j.workspace_id = $2", completed_flow_id, workspace_id, ) diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 505213bb68..7cfe63dc34 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -31,8 +31,7 @@ use windmill_common::{ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ - append_logs, get_queued_job, CanceledBy, JobCompleted, MiniCompletedJob, MiniPulledJob, - ValidableJson, WrappedError, INIT_SCRIPT_TAG, + CanceledBy, INIT_SCRIPT_TAG, JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, WrappedError, append_logs, get_mini_completed_job }; use serde_json::{json, value::RawValue, Value}; @@ -816,9 +815,8 @@ pub async fn handle_job_error( if let Err(err) = updated_flow { if let Some(parent_job_id) = job.parent_job { - // TODO get minicompleted job directly if let Ok(Some(parent_job)) = - get_queued_job(&parent_job_id, &job.workspace_id, &db).await + get_mini_completed_job(&parent_job_id, &job.workspace_id, db).await { let e = json!({"message": err.to_string(), "name": "InternalErr"}); append_logs( @@ -830,7 +828,7 @@ pub async fn handle_job_error( .await; let _ = add_completed_job_error( db, - &MiniCompletedJob::from(MiniPulledJob::from(&parent_job)), + &parent_job, mem_peak, canceled_by.clone(), e, From 605271483365d9a9f59e7196b7364200ad14d33a Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Tue, 4 Nov 2025 16:01:49 +0100 Subject: [PATCH 009/105] fix(ruby): propagate error correctly (#7046) Signed-off-by: pyranota --- backend/windmill-worker/src/ruby_executor.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index 4dbc5bcf8a..cd25663889 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -891,9 +891,23 @@ fn wrap(inner_content: &str) -> Result { require 'json' a = JSON.parse(File.read("args.json")) -res = main(SPREAD) -File.open("result.json", "w") do |file| - file.write(JSON.generate(res)) + +begin + res = main(SPREAD) + File.open("result.json", "w") do |file| + file.write(JSON.generate(res)) + end + +rescue => e + error = { + name: e.class.name, + stack: e.full_message, + message: e.message + } + File.open("result.json", "w") do |file| + file.write(JSON.generate(error)) + end + raise end "# .replace("INNER_CONTENT", inner_content) From bd31f4fc17c914f39853ee8b513ad3d0da560226 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 4 Nov 2025 15:21:16 +0000 Subject: [PATCH 010/105] nit --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 91e3c3f9a0..5a5da8cffc 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ba3429565e0f34c1e764389e950c9797c356ac42 \ No newline at end of file +90bd751d5d103c4484378b9dfaf6a853d0d9bdfc \ No newline at end of file From b0e38dcdade912428e23105af74483e6d0ff77ea Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 4 Nov 2025 15:21:54 +0000 Subject: [PATCH 011/105] nit --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 5a5da8cffc..b8a6f014d2 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -90bd751d5d103c4484378b9dfaf6a853d0d9bdfc \ No newline at end of file +736c4640ff45b9787fc3f2da6ccb15970df439ab \ No newline at end of file From 4a849ca9b96e981bbace020fac7ad3123f330ad3 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 4 Nov 2025 15:26:19 +0000 Subject: [PATCH 012/105] fix: add workspace error handler cache for improved performance --- backend/windmill-queue/src/jobs.rs | 85 +++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 19 deletions(-) diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index f179216d24..5055fa89e5 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -773,6 +773,10 @@ lazy_static::lazy_static! { // Cache for restart_unless_cancelled flag - keyed by (hash, workspace_id) static ref RESTART_UNLESS_CANCELLED_CACHE: Cache<(i64, String), bool> = Cache::new(10000); + + // Cache for workspace error handler settings with 60s TTL + // Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, expiry_timestamp) + static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache, Option>>, bool, i64)> = Cache::new(1000); } pub async fn add_completed_job( @@ -1540,7 +1544,6 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel( } } -//TODO cache all values pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>( queued_job: &MiniCompletedJob, is_canceled: bool, @@ -1549,25 +1552,69 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> ) -> Result<(), Error> { let w_id = &queued_job.workspace_id; - let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( - r#" - SELECT - error_handler, - error_handler_extra_args, - error_handler_muted_on_cancel - FROM - workspace_settings - WHERE - workspace_id = $1 - "#, - ) - .bind(&w_id) - .fetch_optional(db) - .await - .context("fetching error handler info from workspace_settings")? - .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; + // Try to get from cache first, checking if entry is still valid (within 60s TTL) + let now = chrono::Utc::now().timestamp(); + let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel) = + if let Some(cached) = WORKSPACE_ERROR_HANDLER_CACHE.get(w_id) { + if cached.3 > now { + // Cache hit and not expired + (cached.0.clone(), cached.1.clone(), cached.2) + } else { + // Cache expired, fetch from database + let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( + r#" + SELECT + error_handler, + error_handler_extra_args, + error_handler_muted_on_cancel + FROM + workspace_settings + WHERE + workspace_id = $1 + "#, + ) + .bind(&w_id) + .fetch_optional(db) + .await + .context("fetching error handler info from workspace_settings")? + .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; - let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel) = row_result; + // Update cache with 60s TTL + let expiry = now + 60; + WORKSPACE_ERROR_HANDLER_CACHE.insert( + w_id.clone(), + (row_result.0.clone(), row_result.1.clone(), row_result.2, expiry) + ); + row_result + } + } else { + // Cache miss, fetch from database + let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( + r#" + SELECT + error_handler, + error_handler_extra_args, + error_handler_muted_on_cancel + FROM + workspace_settings + WHERE + workspace_id = $1 + "#, + ) + .bind(&w_id) + .fetch_optional(db) + .await + .context("fetching error handler info from workspace_settings")? + .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; + + // Store in cache with 60s TTL + let expiry = now + 60; + WORKSPACE_ERROR_HANDLER_CACHE.insert( + w_id.clone(), + (row_result.0.clone(), row_result.1.clone(), row_result.2, expiry) + ); + row_result + }; if is_canceled && error_handler_muted_on_cancel { return Ok(()); From 8c102aafbdcd6e9caab1585e5adaf3d479073ef3 Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Tue, 4 Nov 2025 16:29:41 +0100 Subject: [PATCH 013/105] nit: fix frontend links for debouncing (#7045) Signed-off-by: pyranota --- frontend/src/lib/components/ScriptBuilder.svelte | 2 +- frontend/src/lib/components/flows/DebounceLimit.svelte | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index c0e0a4b279..a34fcd388a 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -1372,7 +1372,7 @@ {#snippet header()} Debounce Jobs diff --git a/frontend/src/lib/components/flows/DebounceLimit.svelte b/frontend/src/lib/components/flows/DebounceLimit.svelte index c35a8981d8..b7467663cb 100644 --- a/frontend/src/lib/components/flows/DebounceLimit.svelte +++ b/frontend/src/lib/components/flows/DebounceLimit.svelte @@ -41,7 +41,7 @@ options={{ right: 'Debouncing', rightTooltip: 'Consolidate multiple executions into a single run within a time window', - rightDocumentationLink: 'https://www.windmill.dev/docs/core_concepts/debouncing' + rightDocumentationLink: 'https://www.windmill.dev/docs/core_concepts/job_debouncing' }} class="py-1" eeOnly={true} From 0cbb0dacb1df831352fcf1e5bf433bc8cd94c44f Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 4 Nov 2025 16:29:56 +0100 Subject: [PATCH 014/105] parse duckdb json query results (#7040) * parse duckdb json query results * don't pass alias recursively --- .../windmill-duckdb-ffi-internal/Cargo.lock | 6 +- .../windmill-duckdb-ffi-internal/Cargo.toml | 2 +- .../windmill-duckdb-ffi-internal/src/lib.rs | 177 ++++++++++-------- 3 files changed, 106 insertions(+), 79 deletions(-) diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.lock b/backend/windmill-duckdb-ffi-internal/Cargo.lock index b7512f471d..a53a988f30 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.lock +++ b/backend/windmill-duckdb-ffi-internal/Cargo.lock @@ -416,8 +416,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "duckdb" version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a093eed1c714143b257b95fa323e38527fabf05fbf02bb0d5d2045275ffdaef" +source = "git+https://github.com/diegoimbert/duckdb-rs?branch=main#0df52a6941c9996d7ec60d585eaf6430db8c48cf" dependencies = [ "arrow", "cast", @@ -704,8 +703,7 @@ checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libduckdb-sys" version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b93c3ff279601516f01531cadf2ccba50394fbb5f7bf685c6e6b9b07c8dca6f" +source = "git+https://github.com/diegoimbert/duckdb-rs?branch=main#0df52a6941c9996d7ec60d585eaf6430db8c48cf" dependencies = [ "cc", "flate2", diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.toml b/backend/windmill-duckdb-ffi-internal/Cargo.toml index 315055483e..9560e1d0e6 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.toml +++ b/backend/windmill-duckdb-ffi-internal/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] chrono = "0.4.41" -duckdb = { version = "^1.4.1", features = ["bundled"] } +duckdb = { git = "https://github.com/diegoimbert/duckdb-rs", branch = "main", features = ["bundled"] } rust_decimal = "1.37.2" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index cb5c43a018..29112baff9 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -1,11 +1,11 @@ use std::{ collections::HashMap, - ffi::{c_char, CStr, CString}, + ffi::{CStr, CString, c_char}, ptr::null_mut, }; -use duckdb::{params_from_iter, types::TimeUnit, Row}; -use rust_decimal::{prelude::FromPrimitive, Decimal}; +use duckdb::{Row, params_from_iter, types::TimeUnit}; +use rust_decimal::{Decimal, prelude::FromPrimitive}; use serde::Deserialize; use serde_json::value::RawValue; @@ -228,15 +228,16 @@ fn do_duckdb_inner( if skip_collect { return Ok(RawValue::from_string("[]".to_string()).unwrap()); } - // Statement needs to be stepped at least once or stmt.column_names() will panic let mut column_names = None; + let mut type_aliases = None; loop { let row = rows.next(); match row { Ok(Some(row)) => { // Set column names if not already set let stmt = row.as_ref(); + let column_names = match column_names.as_ref() { Some(column_names) => column_names, None => { @@ -244,8 +245,20 @@ fn do_duckdb_inner( column_names.as_ref().unwrap() } }; + let type_aliases = match type_aliases.as_ref() { + Some(type_aliases) => type_aliases, + None => { + type_aliases = Some( + (0..stmt.column_count()) + .map(|i| stmt.column_logical_type(i).get_alias()) + .collect::>(), + ); + type_aliases.as_ref().unwrap() + } + }; - let row = row_to_value(row, &column_names.as_slice()).map_err(|e| e.to_string())?; + let row = row_to_value(row, &column_names.as_slice(), &type_aliases.as_slice()) + .map_err(|e| e.to_string())?; rows_vec.push(row); } Ok(None) => break, @@ -282,83 +295,99 @@ fn interpolate_named_args<'a>( (query, values) } -fn row_to_value(row: &Row<'_>, column_names: &[String]) -> Result, String> { +fn row_to_value( + row: &Row<'_>, + column_names: &[String], + type_aliases: &[Option], +) -> Result, String> { let mut obj = serde_json::Map::new(); for (i, key) in column_names.iter().enumerate() { let value: duckdb::types::Value = row.get(i).map_err(|e| e.to_string())?; - let json_value = match value { - duckdb::types::Value::Null => serde_json::Value::Null, - duckdb::types::Value::Boolean(b) => serde_json::Value::Bool(b), - duckdb::types::Value::TinyInt(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::SmallInt(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::Int(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::BigInt(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::HugeInt(i) => serde_json::Value::String(i.to_string()), - duckdb::types::Value::UTinyInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::USmallInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::UInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::Float(f) => serde_json::Value::Number( - serde_json::Number::from_f64(f as f64) - .ok_or_else(|| "Could not convert to f64".to_string())?, - ), - duckdb::types::Value::Double(f) => serde_json::Value::Number( - serde_json::Number::from_f64(f) - .ok_or_else(|| "Could not convert to f64".to_string())?, - ), - duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), - duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), - duckdb::types::Value::Text(s) => serde_json::Value::String(s), - duckdb::types::Value::Blob(b) => serde_json::Value::Array( - b.into_iter() - .map(|byte| serde_json::Value::Number(byte.into())) - .collect(), - ), - duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()), - duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()), - duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({ - "months": months, - "days": days, - "nanos": nanos - }), - duckdb::types::Value::List(values) => serde_json::Value::Array( - values - .into_iter() - .map(|v| serde_json::Value::String(format!("{:?}", v))) - .collect(), - ), - duckdb::types::Value::Enum(e) => serde_json::Value::String(e), - duckdb::types::Value::Struct(fields) => serde_json::Value::Object( - fields - .iter() - .map(|(k, v)| (k.clone(), serde_json::Value::String(format!("{:?}", v)))) - .collect(), - ), - duckdb::types::Value::Array(values) => serde_json::Value::Array( - values - .into_iter() - .map(|v| serde_json::Value::String(format!("{:?}", v))) - .collect(), - ), - duckdb::types::Value::Map(map) => serde_json::Value::Object( - map.iter() - .map(|(k, v)| { - ( - format!("{:?}", k), - serde_json::Value::String(format!("{:?}", v)), - ) - }) - .collect(), - ), - duckdb::types::Value::Union(value) => { - serde_json::Value::String(format!("{:?}", *value)) - } - }; + let type_alias = &type_aliases[i]; + let json_value = duckdb_value_to_json_value(value, type_alias)?; obj.insert(key.clone(), json_value); } serde_json::value::to_raw_value(&obj).map_err(|e| e.to_string()) } +fn duckdb_value_to_json_value( + value: duckdb::types::Value, + type_alias: &Option, +) -> Result { + let json_value = match value { + duckdb::types::Value::Null => serde_json::Value::Null, + duckdb::types::Value::Boolean(b) => serde_json::Value::Bool(b), + duckdb::types::Value::TinyInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::SmallInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::Int(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::BigInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::HugeInt(i) => serde_json::Value::String(i.to_string()), + duckdb::types::Value::UTinyInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::USmallInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::UInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::Float(f) => serde_json::Value::Number( + serde_json::Number::from_f64(f as f64) + .ok_or_else(|| "Could not convert to f64".to_string())?, + ), + duckdb::types::Value::Double(f) => serde_json::Value::Number( + serde_json::Number::from_f64(f) + .ok_or_else(|| "Could not convert to f64".to_string())?, + ), + duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), + duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), + duckdb::types::Value::Text(s) if type_alias.as_deref().unwrap_or_default() == "JSON" => { + serde_json::from_str(&s) + .map_err(|e| format!("Error parsing JSON text: {}", e.to_string()))? + } + duckdb::types::Value::Text(s) => serde_json::Value::String(s), + duckdb::types::Value::Blob(b) => serde_json::Value::Array( + b.into_iter() + .map(|byte| serde_json::Value::Number(byte.into())) + .collect(), + ), + duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()), + duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()), + duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({ + "months": months, + "days": days, + "nanos": nanos + }), + duckdb::types::Value::List(values) => serde_json::Value::Array( + values + .into_iter() + .map(|v| duckdb_value_to_json_value(v, &None)) + .collect::, _>>()?, + ), + duckdb::types::Value::Enum(e) => serde_json::Value::String(e), + duckdb::types::Value::Struct(fields) => serde_json::Value::Object( + fields + .iter() + .map(|(k, v)| duckdb_value_to_json_value(v.clone(), &None).map(|v| (k.clone(), v))) + .collect::, _>>()?, + ), + duckdb::types::Value::Array(values) => serde_json::Value::Array( + values + .into_iter() + .map(|v| duckdb_value_to_json_value(v, &None)) + .collect::, _>>()?, + ), + duckdb::types::Value::Map(map) => serde_json::Value::Object( + map.iter() + .map(|(k, v)| { + let k = match k { + duckdb::types::Value::Text(s) | duckdb::types::Value::Enum(s) => s.clone(), + _ => format!("{:?}", k), + }; + duckdb_value_to_json_value(v.clone(), &None).map(|v| (k, v)) + }) + .collect::, _>>()?, + ), + duckdb::types::Value::Union(value) => serde_json::Value::String(format!("{:?}", *value)), + }; + Ok(json_value) +} + fn json_value_to_duckdb_value( json_value: &serde_json::Value, arg_type: &str, From 4220582daf23a5367d534c98fb48f0f54ebaab20 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Tue, 4 Nov 2025 10:30:10 -0500 Subject: [PATCH 015/105] fix: pass whitelist env vars to bun install (#7047) --- backend/windmill-worker/src/bun_executor.rs | 1 + backend/windmill-worker/src/worker.rs | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 4e03043c85..757656b934 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -313,6 +313,7 @@ pub async fn install_bun_lockfile( .env_clear() .envs(PROXY_ENVS.clone()) .envs(common_bun_proc_envs) + .envs(&*crate::worker::WHITELIST_ENVS) .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 8cf1e8a7be..db568652e5 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -327,6 +327,12 @@ lazy_static::lazy_static! { } proxy_env }; + pub static ref WHITELIST_ENVS: HashMap = { + windmill_common::worker::load_env_vars( + windmill_common::worker::load_whitelist_env_vars_from_env(), + &HashMap::new(), + ) + }; pub static ref DENO_PATH: String = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string()); pub static ref BUN_PATH: String = std::env::var("BUN_PATH").unwrap_or_else(|_| "/usr/bin/bun".to_string()); pub static ref NPM_PATH: String = std::env::var("NPM_PATH").unwrap_or_else(|_| "/usr/bin/npm".to_string()); From 0ae27a3fe88d6aac046137220aab23e8c0d21590 Mon Sep 17 00:00:00 2001 From: dieriba Date: Tue, 4 Nov 2025 16:39:17 +0100 Subject: [PATCH 016/105] fix: preprocessor schema type (#7049) * fix * Revert "fix" This reverts commit 93618470d06c50a35a676dc180da3c122180c3ba. * ok --- backend/windmill-api/src/triggers/trigger_helpers.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/windmill-api/src/triggers/trigger_helpers.rs b/backend/windmill-api/src/triggers/trigger_helpers.rs index f813f6ab1b..7deaf6dc1d 100644 --- a/backend/windmill-api/src/triggers/trigger_helpers.rs +++ b/backend/windmill-api/src/triggers/trigger_helpers.rs @@ -46,7 +46,7 @@ struct ScriptInfo { #[derive(Debug, Deserialize)] struct PropertyDefinition { - r#type: Option, + r#type: Option>, } #[derive(Debug, Deserialize)] @@ -119,7 +119,11 @@ fn runnable_format_from_schema_without_preprocessor( if schema.as_ref().is_some_and(|schema| { schema.properties.as_ref().is_some_and(|properties| { properties.iter().any(|(key, def)| { - key == "payload" && def.r#type.as_ref().is_some_and(|t| t == "array") + key == "payload" + && def.r#type.as_ref().is_some_and(|t| { + let typ = t.get().trim(); + typ == "array" || (typ.starts_with('[') && typ.ends_with(']')) + }) }) }) }) => From 1d848b2ef7961d160a0a5b77cd4426234d8633a4 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Tue, 4 Nov 2025 10:43:44 -0500 Subject: [PATCH 017/105] git sync repo detection script work with empty repo with no commits (#7039) * fix: git sync repo detection script work with empty repo, no commits + nits * Update frontend/src/lib/components/RepositorySelector.svelte Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> * ee ref --------- Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> --- backend/ee-repo-ref.txt | 2 +- backend/windmill-api/openapi.yaml | 7 +- .../lib/components/RepositorySelector.svelte | 115 ++++++++---------- frontend/src/lib/hubPaths.json | 3 +- 4 files changed, 60 insertions(+), 67 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b8a6f014d2..86814152f7 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -736c4640ff45b9787fc3f2da6ccb15970df439ab \ No newline at end of file +5a4cf98766a6ec5d1b4658366d7f447d3395c122 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 07e37caec8..580a4b9ce4 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -629,12 +629,13 @@ paths: tags: - Git Sync parameters: - - name: search + - name: page in: query - description: Search repositories by name + description: Page number for pagination (default 1) required: false schema: - type: string + type: integer + default: 1 responses: "200": description: connected repositories diff --git a/frontend/src/lib/components/RepositorySelector.svelte b/frontend/src/lib/components/RepositorySelector.svelte index cfe40545c4..ac6f35bbaa 100644 --- a/frontend/src/lib/components/RepositorySelector.svelte +++ b/frontend/src/lib/components/RepositorySelector.svelte @@ -1,7 +1,6 @@
- {#if searchMode} + {#if showSearchableSelect}
= 1 || (searchFilterText.length === 0 && selectedChannel) ? displayChannels().filter(channel => channel.channel_id && channel.channel_name).map((channel) => ({ + items={displayChannels.filter(channel => channel.channel_id && channel.channel_name).map((channel) => ({ label: channel.channel_name ?? 'Unknown Channel', value: channel.channel_id ?? '' - })) : []} + }))} placeholder={isFetching ? "Searching..." : (teamId ? "Search channels..." : "Select a team first")} clearable disabled={disabled || isFetching || !teamId} bind:filterText={searchFilterText} - bind:value={ - () => selectedChannel?.channel_id, - (value) => { - selectedChannel = value ? displayChannels().find((channel) => channel.channel_id === value) : undefined - } - } + bind:value={selectedChannelId} /> {:else} = 1 || (searchFilterText.length === 0 && selectedTeam) ? displayTeams().map((team) => ({ + items={displayTeams.map((team) => ({ label: team.team_name, value: team.team_id - })) : []} + }))} placeholder={isFetching ? "Searching..." : "Search teams..."} clearable disabled={disabled || isFetching} bind:filterText={searchFilterText} - bind:value={ - () => selectedTeam?.team_id, - (value) => { - selectedTeam = value ? displayTeams().find((team) => team.team_id === value) : undefined - } - } + bind:value={selectedTeamId} /> {:else} { + const newKey = e.currentTarget.value.trim() + if (newKey !== entry.key && newKey !== '') { + updateEnvKey(entry.key, newKey) + } + }} + disabled={noEditor} + class="input w-full" + placeholder="VARIABLE_NAME" + /> + +
+ +
+
+ {/each} +
+ {/if} +
+ {#if !noEditor} + + {/if} +
+
+ +
diff --git a/frontend/src/lib/components/flows/map/FlowStickyNode.svelte b/frontend/src/lib/components/flows/map/FlowStickyNode.svelte index 65c68e443b..48e7445593 100644 --- a/frontend/src/lib/components/flows/map/FlowStickyNode.svelte +++ b/frontend/src/lib/components/flows/map/FlowStickyNode.svelte @@ -66,7 +66,7 @@ onClick={() => ($selectedId = 'constants')} /> {#snippet text()} - Static inputs + Environment Variables {/snippet}
{/if} diff --git a/frontend/src/lib/components/flows/previousResults.ts b/frontend/src/lib/components/flows/previousResults.ts index 59fd466b80..1127b0c656 100644 --- a/frontend/src/lib/components/flows/previousResults.ts +++ b/frontend/src/lib/components/flows/previousResults.ts @@ -9,6 +9,7 @@ export type PickableProperties = { priorIds: Record previousId: string | undefined hasResume: boolean + flow_env?: Record } type StepPropPicker = { @@ -156,7 +157,8 @@ export function getFailureStepPropPicker(flowState: FlowState, flow: OpenFlow, a flow_input: schemaToObject(flow.schema as any, args), priorIds: priorIds, previousId: undefined, - hasResume: false + hasResume: false, + flow_env: flow.value.flow_env }, extraLib: ` /** @@ -178,6 +180,17 @@ declare const results = ${JSON.stringify(priorIds)} * flow input as an object */ declare const flow_input = ${JSON.stringify(flowInput)}; + +${ + flow.value.flow_env + ? ` +/** +* flow environment variables +*/ +declare const flow_env = ${JSON.stringify(flow.value.flow_env)}; +` + : '' +} ` } } @@ -218,7 +231,8 @@ export function getStepPropPicker( flow_input: flowInput, priorIds: priorIds, previousId: previousIds[0], - hasResume: previousModule?.suspend != undefined + hasResume: previousModule?.suspend != undefined, + flow_env: flow.value.flow_env } if (pickableProperties.hasResume) { @@ -230,7 +244,8 @@ export function getStepPropPicker( flowInput, priorIds, previousModule?.suspend != undefined, - previousModule?.id + previousModule?.id, + flow.value.flow_env ), pickableProperties } @@ -240,7 +255,8 @@ export function buildExtraLib( flowInput: Record, results: Record, resume: boolean, - previousId: string | undefined + previousId: string | undefined, + flowEnv?: Record ): string { return ` /** @@ -275,6 +291,17 @@ declare const results = ${JSON.stringify(results)}; */ declare const previous_result: ${previousId ? JSON.stringify(results[previousId]) : 'any'}; +${ + flowEnv + ? ` +/** + * flow environment variables + */ +declare const flow_env = ${JSON.stringify(flowEnv)}; +` + : '' +} + ${ resume ? ` diff --git a/frontend/src/lib/components/flows/propPicker/PropPickerWrapper.svelte b/frontend/src/lib/components/flows/propPicker/PropPickerWrapper.svelte index b5346566a2..5d4761d8d5 100644 --- a/frontend/src/lib/components/flows/propPicker/PropPickerWrapper.svelte +++ b/frontend/src/lib/components/flows/propPicker/PropPickerWrapper.svelte @@ -31,6 +31,7 @@ import type { PickableProperties } from '../previousResults' import AnimatedButton from '$lib/components/common/button/AnimatedButton.svelte' import type { PropPickerContext } from '$lib/components/prop_picker' + import type { FlowEditorContext } from '../types' interface Props { pickableProperties: PickableProperties | undefined @@ -69,6 +70,10 @@ const { flowPropPickerConfig } = getContext('PropPickerContext') flowPropPickerConfig.set(undefined) + + const { flowStore } = getContext('FlowEditorContext') + + let flow_env = $derived(pickableProperties?.flow_env || flowStore.val.value.flow_env) setContext('PropPickerWrapper', { propPickerConfig, inputMatches, @@ -156,6 +161,7 @@ | undefined = undefined let variables: Record = {} let resources: Record = {} let displayVariable = false let displayResources = false + let displayFlowEnv = false let allResultsCollapsed = true let collapsableInitialState: @@ -30,6 +32,7 @@ allResultsCollapsed: boolean displayVariable: boolean displayResources: boolean + displayFlowEnv: boolean } | undefined @@ -46,6 +49,7 @@ let flowInputsFiltered: any = pickableProperties.flow_input let resultByIdFiltered: any = pickableProperties.priorIds + let flowEnvFiltered: any = pickableProperties.flow_env let timeout: number | undefined function onSearch(search: string) { @@ -63,6 +67,9 @@ search === EMPTY_STRING ? pickableProperties.priorIds : keepByKey(pickableProperties.priorIds, search) + + flowEnvFiltered = + search === EMPTY_STRING ? pickableProperties.flow_env : keepByKey(pickableProperties.flow_env, search) }, 50) } @@ -98,6 +105,7 @@ if (search === EMPTY_STRING) { flowInputsFiltered = pickableProperties.flow_input resultByIdFiltered = pickableProperties.priorIds + flowEnvFiltered = pickableProperties.flow_env } filteringFlowInputsOrResult = '' return @@ -109,6 +117,9 @@ if (!$inputMatches?.some((match) => match.word === 'results')) { resultByIdFiltered = {} } + if (!$inputMatches?.some((match) => match.word === 'flow_env')) { + flowEnvFiltered = {} + } if ($inputMatches?.length == 1) { filteringFlowInputsOrResult = $inputMatches[0].value if ($inputMatches[0].word === 'flow_input') { @@ -125,6 +136,13 @@ if (Object.keys(filtered).length > 0) { resultByIdFiltered = filtered } + } else if ($inputMatches[0].word === 'flow_env') { + flowEnvFiltered = pickableProperties.flow_env + let [, ...nestedKeys] = $inputMatches[0].value.split('.') + let filtered = filterNestedObject(flowEnvFiltered, nestedKeys) + if (Object.keys(filtered).length > 0) { + flowEnvFiltered = filtered + } } } else { filteringFlowInputsOrResult = '' @@ -143,7 +161,12 @@ } if (!collapsableInitialState) { - collapsableInitialState = { allResultsCollapsed, displayVariable, displayResources } + collapsableInitialState = { + allResultsCollapsed, + displayVariable, + displayResources, + displayFlowEnv + } } if ($inputMatches[0].word === 'variable') { @@ -156,6 +179,10 @@ displayResources = true return } + if ($inputMatches[0].word === 'flow_env') { + displayFlowEnv = true + return + } if ($inputMatches[0].word === 'results') { allResultsCollapsed = false return @@ -166,7 +193,8 @@ if (!collapsableInitialState) { return } - ;({ allResultsCollapsed, displayVariable, displayResources } = collapsableInitialState) + ;({ allResultsCollapsed, displayVariable, displayResources, displayFlowEnv } = + collapsableInitialState) collapsableInitialState = undefined } @@ -183,6 +211,7 @@ if (prev && !filterActive) { flowInputsFiltered = pickableProperties.flow_input resultByIdFiltered = pickableProperties.priorIds + flowEnvFiltered = pickableProperties.flow_env } } @@ -192,7 +221,7 @@ await updateCollapsable() } - $: (search, $inputMatches, $propPickerConfig, pickableProperties, updateState()) + $: search, $inputMatches, $propPickerConfig, pickableProperties, updateState() onDestroy(() => { clearTimeout(timeout) @@ -400,6 +429,45 @@ {/if} {/if} + {#if flow_env && Object.keys(flow_env).length > 0 && (!filterActive || $inputMatches?.some((match) => match.word === 'flow_env'))} +
+ Flow Env Variables: + + {#if displayFlowEnv} + + + {:else} + + {/if} +
+ {/if} {/if} diff --git a/frontend/src/lib/components/propertyPicker/PropPickerResult.svelte b/frontend/src/lib/components/propertyPicker/PropPickerResult.svelte index cbb2f9c33c..7d56abeb05 100644 --- a/frontend/src/lib/components/propertyPicker/PropPickerResult.svelte +++ b/frontend/src/lib/components/propertyPicker/PropPickerResult.svelte @@ -5,6 +5,7 @@ export let result: any export let extraResults: any = undefined export let flow_input: any = undefined + export let flow_env: any = undefined
@@ -18,4 +19,10 @@
{/if} + {#if flow_env} + Flow Environment Variables +
+ +
+ {/if} diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index f332964c40..a9a4009211 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -62,6 +62,10 @@ components: type: string cache_ttl: type: number + flow_env: + type: object + additionalProperties: + type: string priority: type: number early_return: From 3a657b10e78466d6af386faa9fee6303b190d813 Mon Sep 17 00:00:00 2001 From: dieriba Date: Fri, 7 Nov 2025 19:51:22 +0100 Subject: [PATCH 063/105] nit flow env (#7090) --- backend/windmill-worker/src/js_eval.rs | 67 ++++++++++--------- .../propertyPicker/PropPicker.svelte | 2 +- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index ec28b6f509..5dd3f086d8 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -220,36 +220,40 @@ fn try_exact_property_access( } async fn handle_full_regex( - captures: regex::Captures<'_>, + expr: &str, authed_client: &AuthedClient, by_id: &IdContext, -) -> anyhow::Result> { - let obj_name = captures.get(1).unwrap().as_str(); - let obj_key = captures.get(2).unwrap().as_str(); - let idx_o = captures.get(3).map(|y| y.as_str()); - let rest = captures.get(4).map(|y| y.as_str()); - let query = if let Some(idx) = idx_o { - match rest { - Some(rest) => Some(format!("{}{}", idx, rest)), - None => Some(idx.to_string()), - } - } else { - rest.map(|x| x.trim_start_matches('.').to_string()) - }; +) -> Option>> { + if let Some(captures) = RE_FULL.captures(&expr) { + let obj_name = captures.get(1).unwrap().as_str(); + let obj_key = captures.get(2).unwrap().as_str(); + let idx_o = captures.get(3).map(|y| y.as_str()); + let rest = captures.get(4).map(|y| y.as_str()); + let query = if let Some(idx) = idx_o { + match rest { + Some(rest) => Some(format!("{}{}", idx, rest)), + None => Some(idx.to_string()), + } + } else { + rest.map(|x| x.trim_start_matches('.').to_string()) + }; - let result = if obj_name == "results" { - authed_client - .get_result_by_id(&by_id.flow_job.to_string(), obj_key, query) - .await - } else if obj_name == "flow_env" { - authed_client - .get_flow_env_by_flow_job_id(&by_id.flow_job.to_string(), obj_key, query) - .await - } else { - unreachable!(); - }; + let result = if obj_name == "results" { + authed_client + .get_result_by_id(&by_id.flow_job.to_string(), obj_key, query) + .await + } else if obj_name == "flow_env" { + authed_client + .get_flow_env_by_flow_job_id(&by_id.flow_job.to_string(), obj_key, query) + .await + } else { + unreachable!(); + }; - return result; + return Some(result); + } + + return None; } pub async fn eval_timeout( @@ -299,8 +303,8 @@ pub async fn eval_timeout( } if let (Some(by_id), Some(authed_client)) = (by_id, authed_client) { - if let Some(captures) = RE_FULL.captures(&expr) { - return handle_full_regex(captures, authed_client, by_id).await; + if let Some(result) = handle_full_regex(&expr, authed_client, by_id).await { + return result; } } @@ -444,9 +448,10 @@ fn replace_with_await(expr: String, fn_name: &str) -> String { s } lazy_static! { - static ref RE: Regex = - Regex::new(r#"(?m)(?P(?:results|flow_env)(?:\?)?(?:(?:\.[a-zA-Z_0-9]+)|(?:\[\".*?\"\])))"#) - .unwrap(); + static ref RE: Regex = Regex::new( + r#"(?m)(?P(?:results|flow_env)(?:\?)?(?:(?:\.[a-zA-Z_0-9]+)|(?:\[\".*?\"\])))"# + ) + .unwrap(); static ref RE_FULL: Regex = Regex::new( r"(?m)^(results|flow_env)(?:\?)?\.([a-zA-Z_0-9]+)(?:\[(\d+)\])?((?:\.[a-zA-Z_0-9]+)+)?$" ) diff --git a/frontend/src/lib/components/propertyPicker/PropPicker.svelte b/frontend/src/lib/components/propertyPicker/PropPicker.svelte index af42ecb864..c90a437b8a 100644 --- a/frontend/src/lib/components/propertyPicker/PropPicker.svelte +++ b/frontend/src/lib/components/propertyPicker/PropPicker.svelte @@ -18,7 +18,7 @@ export let error: boolean = false export let allowCopy = false export let previousId: string | undefined = undefined - export let flow_env: Record | undefined = undefined + export let flow_env: Record | undefined = undefined let variables: Record = {} let resources: Record = {} From 408911dbf6a7ee110bbb66ffcf127804ff253989 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 7 Nov 2025 13:51:33 -0500 Subject: [PATCH 064/105] camelcase (#7091) --- frontend/src/lib/components/ChannelSelector.svelte | 6 +++--- frontend/src/lib/components/ErrorOrRecoveryHandler.svelte | 2 +- frontend/src/lib/components/InstanceSetting.svelte | 4 ++-- frontend/src/lib/components/TeamSelector.svelte | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/frontend/src/lib/components/ChannelSelector.svelte b/frontend/src/lib/components/ChannelSelector.svelte index b6e77e06d9..7bca9d26d0 100644 --- a/frontend/src/lib/components/ChannelSelector.svelte +++ b/frontend/src/lib/components/ChannelSelector.svelte @@ -18,7 +18,7 @@ channels?: ChannelItem[] teamId?: string onError?: (error: Error) => void - onselectedchannelchange?: (channel: ChannelItem | undefined) => void + onSelectedChannelChange?: (channel: ChannelItem | undefined) => void } let { @@ -30,7 +30,7 @@ channels = undefined, teamId, onError, - onselectedchannelchange + onSelectedChannelChange }: Props = $props() let isFetching = $state(false) @@ -69,7 +69,7 @@ $effect(() => { if (selectedChannel?.channel_id !== previousChannelId) { previousChannelId = selectedChannel?.channel_id - onselectedchannelchange?.(selectedChannel) + onSelectedChannelChange?.(selectedChannel) } }) diff --git a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte index 25bc2f9adf..a21307977b 100644 --- a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte +++ b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte @@ -502,7 +502,7 @@ channel_name: handlerExtraArgs['channel_name'] } : undefined} - onselectedchannelchange={(channel) => { + onSelectedChannelChange={(channel) => { handlerExtraArgs['channel'] = channel?.channel_id handlerExtraArgs['channel_name'] = channel?.channel_name }} diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 92685107a9..9e5906fdd9 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -586,7 +586,7 @@ minWidth="140px" showRefreshButton={false} selectedTeam={currentTeam} - onselectedteamchange={(team) => handleTeamChange(team, i)} + onSelectedTeamChange={(team) => handleTeamChange(team, i)} /> {#if $values['critical_error_channels'][i]?.teams_channel?.team_id} @@ -595,7 +595,7 @@ placeholder="Search channels" teamId={$values['critical_error_channels'][i]?.teams_channel?.team_id} selectedChannel={currentChannel} - onselectedchannelchange={(channel) => handleChannelChange(channel, i)} + onSelectedChannelChange={(channel) => handleChannelChange(channel, i)} onError={(e) => sendUserToast('Failed to load channels: ' + e.message, true)} /> diff --git a/frontend/src/lib/components/TeamSelector.svelte b/frontend/src/lib/components/TeamSelector.svelte index e55f0bec8f..6ba222f12d 100644 --- a/frontend/src/lib/components/TeamSelector.svelte +++ b/frontend/src/lib/components/TeamSelector.svelte @@ -18,7 +18,7 @@ teams?: TeamItem[] | undefined minWidth?: string onError?: (error: Error) => void - onselectedteamchange?: (team: TeamItem | undefined) => void + onSelectedTeamChange?: (team: TeamItem | undefined) => void } let { @@ -29,7 +29,7 @@ teams = undefined, minWidth = '160px', onError, - onselectedteamchange + onSelectedTeamChange }: Props = $props() let isFetching = $state(false) @@ -68,7 +68,7 @@ $effect(() => { if (selectedTeam?.team_id !== previousTeamId) { previousTeamId = selectedTeam?.team_id - onselectedteamchange?.(selectedTeam) + onSelectedTeamChange?.(selectedTeam) } }) From 62ffe9ffce6d070c81794e8483ab83b431aee9ea Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 7 Nov 2025 23:10:21 +0000 Subject: [PATCH 065/105] fix(cli): add automatic handler of .node files for codebase bundler --- cli/src/commands/script/script.ts | 68 +++++++++++++++++++++++++------ cli/src/core/conf.ts | 1 + cli/src/utils/codebase.ts | 15 ++++--- 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 3903be6f71..1c95cb297d 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -180,6 +180,14 @@ export async function handleScriptMetadata( } } +export interface OutputFile { + path: string + contents: Uint8Array + hash: string + /** "contents" as text (changes automatically with "contents") */ + readonly text: string +} + export async function handleFile( path: string, workspace: Workspace, @@ -210,7 +218,9 @@ export async function handleFile( let bundleContent: string | Tarball | undefined = undefined; + let forceTar = false; if (codebase) { + let outputFiles: OutputFile[] = []; if (codebase.customBundler) { log.info(`Using custom bundler ${codebase.customBundler} for ${path}`); bundleContent = execSync( @@ -232,35 +242,42 @@ export async function handleFile( external: codebase.external, inject: codebase.inject, define: codebase.define, + loader: codebase.loader ?? { ".node": "file" }, + outdir: '/', platform: "node", packages: "bundle", target: format == "cjs" ? "node20.15.1" : "esnext", }); const endTime = performance.now(); bundleContent = out.outputFiles[0].text; + outputFiles = out.outputFiles; log.info( `Finished bundling ${path}: ${(bundleContent.length / 1024).toFixed( 0 )}kB (${(endTime - startTime).toFixed(0)}ms)` ); } - if (Array.isArray(codebase.assets) && codebase.assets.length > 0) { + if (outputFiles.length > 1) { const archiveNpm = await import("npm:@ayonli/jsext/archive"); log.info( - `Using the following asset configuration for ${path}: ${JSON.stringify( - codebase.assets - )}` + `Found multiple output files for ${path}, creating a tarball... ${outputFiles.map((file) => file.path).join(", ")}` ); + forceTar = true; const startTime = performance.now(); const tarball = new archiveNpm.Tarball(); + const mainPath = path.split(SEP).pop()?.split(".")[0] + ".js"; + const content = outputFiles.find((file) => file.path == "/" + mainPath)?.text ?? ''; + log.info(`Main content: ${content.length}chars`); tarball.append( - new File([bundleContent], "main.js", { type: "text/plain" }) + new File([content], "main.js", { type: "text/plain" }) ); - for (const asset of codebase.assets) { - const data = fs.readFileSync(asset.from); - const blob = new Blob([data], { type: "text/plain" }); - const file = new File([blob], asset.to); - tarball.append(file); + for (const file of outputFiles) { + if (file.path == "/" + mainPath) { + continue; + } + log.info(`Adding file: ${file.path.substring(1)}`); + const fil = new File([file.contents], file.path.substring(1)); + tarball.append(fil); } const endTime = performance.now(); log.info( @@ -269,6 +286,33 @@ export async function handleFile( ).toFixed(0)}kB (${(endTime - startTime).toFixed(0)}ms)` ); bundleContent = tarball; + } else { + if (Array.isArray(codebase.assets) && codebase.assets.length > 0) { + const archiveNpm = await import("npm:@ayonli/jsext/archive"); + log.info( + `Using the following asset configuration for ${path}: ${JSON.stringify( + codebase.assets + )}` + ); + const startTime = performance.now(); + const tarball = new archiveNpm.Tarball(); + tarball.append( + new File([bundleContent], "main.js", { type: "text/plain" }) + ); + for (const asset of codebase.assets) { + const data = fs.readFileSync(asset.from); + const blob = new Blob([data], { type: "text/plain" }); + const file = new File([blob], asset.to); + tarball.append(file); + } + const endTime = performance.now(); + log.info( + `Finished creating tarball for ${path}: ${( + tarball.size / 1024 + ).toFixed(0)}kB (${(endTime - startTime).toFixed(0)}ms)` + ); + bundleContent = tarball; + } } } let typed = opts?.skipScriptsMetadata @@ -325,7 +369,7 @@ export async function handleFile( } if (typed && codebase) { - typed.codebase = await codebase.getDigest(); + typed.codebase = await codebase.getDigest(forceTar); } const requestBodyCommon: NewScript = { @@ -352,7 +396,7 @@ export async function handleFile( concurrency_key: typed?.concurrency_key, debounce_key: typed?.debounce_key, debounce_delay_s: typed?.debounce_delay_s, - codebase: await codebase?.getDigest(), + codebase: await codebase?.getDigest(forceTar), timeout: typed?.timeout, on_behalf_of_email: typed?.on_behalf_of_email, }; diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 5a56f97435..0a6dbe8167 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -99,6 +99,7 @@ export interface Codebase { external?: string[]; define?: { [key: string]: string }; inject?: string[]; + loader?: any, format?: "cjs" | "esm"; } diff --git a/cli/src/utils/codebase.ts b/cli/src/utils/codebase.ts index 36dfbb16ed..3391f6f795 100644 --- a/cli/src/utils/codebase.ts +++ b/cli/src/utils/codebase.ts @@ -2,7 +2,7 @@ import { Codebase, SyncOptions } from "../core/conf.ts"; import { log } from "../../deps.ts"; import { digestDir } from "./utils.ts"; -export type SyncCodebase = Codebase & { getDigest: () => Promise }; +export type SyncCodebase = Codebase & { getDigest: (forceTar?: boolean) => Promise }; export function listSyncCodebases( options: SyncOptions ): SyncCodebase[] { @@ -13,16 +13,21 @@ export function listSyncCodebases( } for (const codebase of options?.codebases ?? []) { let _digest: string | undefined = undefined; - const getDigest: () => Promise = async () => { - if (_digest == undefined) { + let alreadyPrinted = false; + const getDigest: (forceTar?: boolean) => Promise = async (forceTar?: boolean) => { + if (_digest == undefined || forceTar) { _digest = await digestDir( codebase.relative_path, JSON.stringify(codebase) ); - if (Array.isArray(codebase.assets) && codebase.assets.length > 0) { + if (forceTar || (Array.isArray(codebase.assets) && codebase.assets.length > 0)) { _digest += ".tar"; } - log.info(`Codebase ${codebase.relative_path}, digest: ${_digest}`); + if (!alreadyPrinted) { + alreadyPrinted = true; + log.info(`Codebase ${codebase.relative_path}, digest: ${_digest}`); + } + return _digest; } return _digest; }; From b5c21cfe560b4d37124f08ebdfae5a5340464b9e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 8 Nov 2025 00:27:45 +0100 Subject: [PATCH 066/105] chore(main): release 1.574.0 (#7088) * chore(main): release 1.574.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 13 ++++ backend/Cargo.lock | 62 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 61 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e20b7f4631..a016172a01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.574.0](https://github.com/windmill-labs/windmill/compare/v1.573.5...v1.574.0) (2025-11-07) + + +### Features + +* env var in flow ([#6852](https://github.com/windmill-labs/windmill/issues/6852)) ([c59183f](https://github.com/windmill-labs/windmill/commit/c59183f5c39f853d9679c00dd5aa755ef171d735)) + + +### Bug Fixes + +* **cli:** add automatic handler of .node files for codebase bundler ([62ffe9f](https://github.com/windmill-labs/windmill/commit/62ffe9ffce6d070c81794e8483ab83b431aee9ea)) +* teams selector svelte5 ([#7087](https://github.com/windmill-labs/windmill/issues/7087)) ([6045f0c](https://github.com/windmill-labs/windmill/commit/6045f0c40654a88e93be688bcdfab874cfc0b267)) + ## [1.573.5](https://github.com/windmill-labs/windmill/compare/v1.573.4...v1.573.5) (2025-11-07) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 0485c7aa07..51f3658734 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -9071,9 +9071,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.74" +version = "0.10.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24ad14dd45412269e1a30f52ad8f0664f0f4f4a89ee8fe28c3b3527021ebb654" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ "bitflags 2.9.4", "cfg-if", @@ -9112,9 +9112,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.110" +version = "0.9.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a9f0075ba3c21b09f8e8b2026584b1d18d49388648f2fbbf3c97ea8deced8e2" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" dependencies = [ "cc", "libc", @@ -15137,7 +15137,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "aws-sdk-config", @@ -15197,7 +15197,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "argon2", @@ -15317,7 +15317,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.573.5" +version = "1.574.0" dependencies = [ "base64 0.22.1", "chrono", @@ -15332,7 +15332,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.573.5" +version = "1.574.0" dependencies = [ "chrono", "lazy_static", @@ -15346,7 +15346,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "axum", @@ -15365,7 +15365,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "async-recursion", @@ -15450,7 +15450,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.573.5" +version = "1.574.0" dependencies = [ "regex", "serde", @@ -15465,7 +15465,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "bytes", @@ -15489,7 +15489,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.573.5" +version = "1.574.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15501,7 +15501,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.573.5" +version = "1.574.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15510,7 +15510,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "lazy_static", @@ -15522,7 +15522,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "serde_json", @@ -15534,7 +15534,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "gosyn", @@ -15546,7 +15546,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "lazy_static", @@ -15558,7 +15558,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "serde_json", @@ -15570,7 +15570,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "nu-parser", @@ -15581,7 +15581,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15592,7 +15592,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15604,7 +15604,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "async-recursion", @@ -15627,7 +15627,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "lazy_static", @@ -15641,7 +15641,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15658,7 +15658,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "lazy_static", @@ -15672,7 +15672,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "lazy_static", @@ -15690,7 +15690,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "serde", @@ -15701,7 +15701,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "async-recursion", @@ -15735,7 +15735,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.573.5" +version = "1.574.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15745,7 +15745,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.573.5" +version = "1.574.0" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 26cacfdb8a..ebe21ebbbe 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.573.5" +version = "1.574.0" authors.workspace = true edition.workspace = true @@ -33,7 +33,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.573.5" +version = "1.574.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 3d666ec4df..b393ab4eef 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.573.5 + version: 1.574.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 5dfc44b9ae..34da382ba2 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.573.5"; +export const VERSION = "v1.574.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 413cd2a59a..9c92ab9423 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.573.5"; +export const VERSION = "1.574.0"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 48eea0020c..790fa113f0 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.573.5", + "version": "1.574.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.573.5", + "version": "1.574.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 47b0b78bc8..efdae145cb 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.573.5", + "version": "1.574.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 0a2289344c..f8abc440f7 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.573.5" -wmill_pg = ">=1.573.5" +wmill = ">=1.574.0" +wmill_pg = ">=1.574.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index a9a4009211..0faa3b8db6 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.573.5 + version: 1.574.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index cb51f183c0..117b970c97 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.573.5' + ModuleVersion = '1.574.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 8de01a20a3..190f6f960c 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.573.5" +version = "1.574.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 6410052700..487d2d1b71 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.573.5" +version = "1.574.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 4f6cf29147..ca9a8bfd74 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.573.5", + "version": "1.574.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 44676689ad..71017cdfc9 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.573.5", + "version": "1.574.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 0ace6d04cf..612cab4205 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.573.5 +1.574.0 From d6421c2ea79993ef7815c50cf035d3b3425e271e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 8 Nov 2025 00:20:49 +0000 Subject: [PATCH 067/105] fix: make get_logs work even for partial flow jobs --- ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...02b9899b4a97b8e557692c0085a9b472b8a7.json} | 8 +++---- backend/windmill-api/src/jobs.rs | 22 +++++++++++++++---- 3 files changed, 23 insertions(+), 9 deletions(-) rename backend/.sqlx/{query-35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b.json => query-5e7cadffbee74b11e224b60322b102b9899b4a97b8e557692c0085a9b472b8a7.json} (57%) diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b.json b/backend/.sqlx/query-5e7cadffbee74b11e224b60322b102b9899b4a97b8e557692c0085a9b472b8a7.json similarity index 57% rename from backend/.sqlx/query-35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b.json rename to backend/.sqlx/query-5e7cadffbee74b11e224b60322b102b9899b4a97b8e557692c0085a9b472b8a7.json index e3d94ad1cf..414ba5924d 100644 --- a/backend/.sqlx/query-35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b.json +++ b/backend/.sqlx/query-5e7cadffbee74b11e224b60322b102b9899b4a97b8e557692c0085a9b472b8a7.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_job j\n LEFT JOIN job_logs ON job_logs.job_id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", + "query": "SELECT j.created_by AS \"created_by\", coalesce(job_logs.logs, '') as logs, COALESCE(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_job j\n LEFT JOIN job_logs ON job_logs.job_id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { "ordinal": 0, - "name": "created_by!", + "name": "created_by", "type_info": "Varchar" }, { @@ -34,9 +34,9 @@ "nullable": [ false, null, - false, + null, true ] }, - "hash": "35061719d01929a7146c80de4b637abdad3198d3340ec7c04ed671baff0a4d0b" + "hash": "5e7cadffbee74b11e224b60322b102b9899b4a97b8e557692c0085a9b472b8a7" } diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index d2de1a8f49..d006c2ec15 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -1468,7 +1468,7 @@ async fn get_job_logs( .flatten(); let record = sqlx::query!( - "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index + "SELECT j.created_by AS \"created_by\", coalesce(job_logs.logs, '') as logs, COALESCE(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index FROM v2_job j LEFT JOIN job_logs ON job_logs.job_id = j.id WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", @@ -1497,11 +1497,21 @@ async fn get_job_logs( .await?; #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(r) = get_logs_from_store(record.log_offset, &logs, &record.log_file_index).await + if let Some(r) = get_logs_from_store( + record.log_offset.unwrap_or(0), + &logs, + &record.log_file_index, + ) + .await { return r.map(content_plain); } - if let Some(r) = get_logs_from_disk(record.log_offset, &logs, &record.log_file_index).await + if let Some(r) = get_logs_from_disk( + record.log_offset.unwrap_or(0), + &logs, + &record.log_file_index, + ) + .await { return r.map(content_plain); } @@ -4285,7 +4295,11 @@ pub async fn run_script_by_path_inner( timeout, None, // If the job has a parent job, set priority to 2 as it may be ran synchronously and block a current worker until being executed. Flow steps have a priority of 1 so this is higher. - if run_query.parent_job.is_some() || run_query.root_job.is_some() { Some(2) } else { None }, + if run_query.parent_job.is_some() || run_query.root_job.is_some() { + Some(2) + } else { + None + }, push_authed.as_ref(), false, None, From f12be4eb190d9fbb218e3f91f2f2b9a3852faee5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 8 Nov 2025 10:26:48 +0000 Subject: [PATCH 068/105] fix direct access --- backend/windmill-worker/src/js_eval.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 5dd3f086d8..5827c8ba54 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -192,7 +192,7 @@ fn try_exact_property_access( let suffix = &expr[access_pattern_pos..]; let maybe_key_name = if suffix.starts_with(DOT_PATTERN) { let key_name_pos = DOT_PATTERN.len(); - Some(&expr[key_name_pos..]) + Some(&suffix[key_name_pos..]) } else if suffix.starts_with(START_BRACKET_PATTERN) { let key_name_pos = START_BRACKET_PATTERN.len(); let suffix = &suffix[key_name_pos..]; From 8b2291b0f908075a6793feaa92a094c09bd45253 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 8 Nov 2025 11:28:55 +0100 Subject: [PATCH 069/105] chore(main): release 1.574.1 (#7092) * chore(main): release 1.574.1 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 7 +++ backend/Cargo.lock | 54 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 51 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a016172a01..acc2bd6c5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.574.1](https://github.com/windmill-labs/windmill/compare/v1.574.0...v1.574.1) (2025-11-08) + + +### Bug Fixes + +* make get_logs work even for partial flow jobs ([d6421c2](https://github.com/windmill-labs/windmill/commit/d6421c2ea79993ef7815c50cf035d3b3425e271e)) + ## [1.574.0](https://github.com/windmill-labs/windmill/compare/v1.573.5...v1.574.0) (2025-11-07) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 51f3658734..cb19092a20 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15137,7 +15137,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "aws-sdk-config", @@ -15197,7 +15197,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "argon2", @@ -15317,7 +15317,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.574.0" +version = "1.574.1" dependencies = [ "base64 0.22.1", "chrono", @@ -15332,7 +15332,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.574.0" +version = "1.574.1" dependencies = [ "chrono", "lazy_static", @@ -15346,7 +15346,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "axum", @@ -15365,7 +15365,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "async-recursion", @@ -15450,7 +15450,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.574.0" +version = "1.574.1" dependencies = [ "regex", "serde", @@ -15465,7 +15465,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "bytes", @@ -15489,7 +15489,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.574.0" +version = "1.574.1" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15501,7 +15501,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.574.0" +version = "1.574.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -15510,7 +15510,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "lazy_static", @@ -15522,7 +15522,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "serde_json", @@ -15534,7 +15534,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "gosyn", @@ -15546,7 +15546,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "lazy_static", @@ -15558,7 +15558,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "serde_json", @@ -15570,7 +15570,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "nu-parser", @@ -15581,7 +15581,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15592,7 +15592,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15604,7 +15604,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "async-recursion", @@ -15627,7 +15627,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "lazy_static", @@ -15641,7 +15641,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15658,7 +15658,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "lazy_static", @@ -15672,7 +15672,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "lazy_static", @@ -15690,7 +15690,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "serde", @@ -15701,7 +15701,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "async-recursion", @@ -15735,7 +15735,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.574.0" +version = "1.574.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15745,7 +15745,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.574.0" +version = "1.574.1" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ebe21ebbbe..41e33efd9a 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.574.0" +version = "1.574.1" authors.workspace = true edition.workspace = true @@ -33,7 +33,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.574.0" +version = "1.574.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index b393ab4eef..47f9466a06 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.574.0 + version: 1.574.1 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 34da382ba2..14179cdde7 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.574.0"; +export const VERSION = "v1.574.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 9c92ab9423..88ef77d73b 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.574.0"; +export const VERSION = "1.574.1"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 790fa113f0..2055f79b82 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.574.0", + "version": "1.574.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.574.0", + "version": "1.574.1", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index efdae145cb..3f5a13c88e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.574.0", + "version": "1.574.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index f8abc440f7..5207532db2 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.574.0" -wmill_pg = ">=1.574.0" +wmill = ">=1.574.1" +wmill_pg = ">=1.574.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 0faa3b8db6..156eac8a54 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.574.0 + version: 1.574.1 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 117b970c97..ce25df0d21 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.574.0' + ModuleVersion = '1.574.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 190f6f960c..7453c8d7de 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.574.0" +version = "1.574.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 487d2d1b71..cd9d5aecc8 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.574.0" +version = "1.574.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index ca9a8bfd74..1ded4087d4 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.574.0", + "version": "1.574.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 71017cdfc9..1bc35510dc 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.574.0", + "version": "1.574.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 612cab4205..50a7825cc3 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.574.0 +1.574.1 From 9e4882c0a919303480f5a431db3c05b1d5855d37 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 8 Nov 2025 19:05:23 +0000 Subject: [PATCH 070/105] fix: fix multiselect in list for apps --- .../apps/components/inputs/AppMultiSelectV2.svelte | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/apps/components/inputs/AppMultiSelectV2.svelte b/frontend/src/lib/components/apps/components/inputs/AppMultiSelectV2.svelte index 929e4cb147..21171e3e11 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppMultiSelectV2.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppMultiSelectV2.svelte @@ -55,7 +55,11 @@ }) let selectedItems: string[] = $state([...new Set(outputs?.result.peak())].map(convertToValue)) - $effect(() => setResultsFromSelectedItems(selectedItems)) + $effect(() => { + selectedItems + // console.log('selectedItems', selectedItems) + untrack(() => setResultsFromSelectedItems(selectedItems)) + }) let customItems: string[] = $state([]) From 2d54dfbf05f21faa14655b7124daa291a9d8d66d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 8 Nov 2025 19:49:47 +0000 Subject: [PATCH 071/105] fix: make ai chat works with unicode messages --- .../conversations/FlowChatManager.svelte.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts index 8afd1708e5..edde28bbef 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts +++ b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts @@ -371,17 +371,20 @@ class FlowChatManager { let isCompleted = false try { + const jobId = await JobService.runFlowByPath({ + workspace: this.#workspace!, + path: this.#path!, + requestBody: { user_message: messageContent }, + memoryId: currentConversationId + }) // Encode the payload as base64 - const payload = { user_message: messageContent } - const payloadBase64 = btoa(JSON.stringify(payload)) // Build the EventSource URL - const streamUrl = `/api/w/${this.#workspace}/jobs/run_and_stream/f/${this.#path}` + const streamUrl = `/api/w/${this.#workspace}/jobs_u/getupdate_sse/${jobId}` const url = new URL(streamUrl, window.location.origin) - url.searchParams.set('payload', payloadBase64) - url.searchParams.set('memory_id', currentConversationId) url.searchParams.set('poll_delay_ms', '50') - + url.searchParams.set('fast', 'true') + url.searchParams.set('only_result', 'true') // Create EventSource connection const eventSource = new EventSource(url.toString()) this.currentEventSource = eventSource @@ -392,7 +395,7 @@ class FlowChatManager { eventSource.onmessage = async (event) => { try { const data = JSON.parse(event.data) - + console.log('data', data) if (data.type === 'update') { if (data.flow_stream_job_id) { this.currentJobId = data.flow_stream_job_id From 41a6f89bdbd29c4635c083d8fa4968ab8b76a4a2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 8 Nov 2025 20:14:07 +0000 Subject: [PATCH 072/105] not require crypto for ai chat --- .../src/lib/components/FlowPreviewContent.svelte | 3 ++- .../components/flows/content/FlowInput.svelte | 3 ++- .../conversations/FlowChatManager.svelte.ts | 16 ++++++++++++---- .../(logged)/flows/get/[...path]/+page.svelte | 3 ++- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 3c5267917d..cab32403f3 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -39,6 +39,7 @@ import { aiChatManager } from './copilot/chat/AIChatManager.svelte' import { stateSnapshot } from '$lib/svelte5Utils.svelte' import FlowChatInterface from './flows/conversations/FlowChatInterface.svelte' + import { randomUUID } from './flows/conversations/FlowChatManager.svelte' interface Props { previewMode: 'upTo' | 'whole' @@ -469,7 +470,7 @@ return jobId ?? '' }} createConversation={async () => { - const newConversationId = crypto.randomUUID() + const newConversationId = randomUUID() return newConversationId }} /> diff --git a/frontend/src/lib/components/flows/content/FlowInput.svelte b/frontend/src/lib/components/flows/content/FlowInput.svelte index 7eae608898..bffdd2dcf9 100644 --- a/frontend/src/lib/components/flows/content/FlowInput.svelte +++ b/frontend/src/lib/components/flows/content/FlowInput.svelte @@ -49,6 +49,7 @@ import { AI_AGENT_SCHEMA } from '../flowInfers' import { nextId } from '../flowModuleNextId' import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte' + import { randomUUID } from '../conversations/FlowChatManager.svelte' interface Props { noEditor: boolean @@ -491,7 +492,7 @@ { - const newConversationId = crypto.randomUUID() + const newConversationId = randomUUID() return newConversationId }} /> diff --git a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts index edde28bbef..8f0948f4ed 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts +++ b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts @@ -18,6 +18,15 @@ export interface FlowChatManagerOptions { path?: string } +export function randomUUID() { + // Pure JS (RFC4122 v4) UUID implementation (no external dependencies) + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { + const r = (Math.random() * 16) | 0 + const v = c === 'x' ? r : (r & 0x3) | 0x8 + return v.toString(16) + }) +} + class FlowChatManager { // State messages = $state([]) @@ -320,7 +329,7 @@ class FlowChatManager { delete this.#conversationsCache[currentConversationId] const userMessage: ChatMessage = { - id: crypto.randomUUID(), + id: randomUUID(), content: this.inputMessage.trim(), created_at: new Date().toISOString(), message_type: 'user', @@ -395,7 +404,6 @@ class FlowChatManager { eventSource.onmessage = async (event) => { try { const data = JSON.parse(event.data) - console.log('data', data) if (data.type === 'update') { if (data.flow_stream_job_id) { this.currentJobId = data.flow_stream_job_id @@ -423,7 +431,7 @@ class FlowChatManager { this.messages = [ ...this.messages, { - id: 'temp-' + crypto.randomUUID(), + id: 'temp-' + randomUUID(), content: newContent, created_at: new Date().toISOString(), message_type: 'tool', @@ -445,7 +453,7 @@ class FlowChatManager { assistantMessageId.length === 0 && accumulatedContent.length > 0 ) { - assistantMessageId = 'temp-' + crypto.randomUUID() + assistantMessageId = 'temp-' + randomUUID() this.messages = [ ...this.messages, { diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 84ceabe2b6..444bed9496 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -65,6 +65,7 @@ initFlowGraphAssetsCtx } from '$lib/components/flows/FlowAssetsHandler.svelte' import { page } from '$app/state' + import { randomUUID } from '$lib/components/flows/conversations/FlowChatManager.svelte' let flow: Flow | undefined = $state() let can_write = false @@ -401,7 +402,7 @@ let path = $derived(page.params.path ?? '') async function handleNewConversation({ clearMessages = true }: { clearMessages?: boolean }) { - const newConversationId = crypto.randomUUID() + const newConversationId = randomUUID() // Add the new conversation to the sidebar (returns id of draft or new conversation) if (flowConversationsSidebar) { From 3dcad57481a2dcc97694692d83806d009167d4ac Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 9 Nov 2025 10:53:37 +0000 Subject: [PATCH 073/105] add debug_sse_stream --- backend/windmill-worker/src/ai/sse.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/windmill-worker/src/ai/sse.rs b/backend/windmill-worker/src/ai/sse.rs index fcafc9cd7e..42c5d73603 100644 --- a/backend/windmill-worker/src/ai/sse.rs +++ b/backend/windmill-worker/src/ai/sse.rs @@ -41,6 +41,12 @@ pub struct OpenAISSEEvent { pub choices: Option>, } +lazy_static::lazy_static! { + static ref DEBUG_SSE_STREAM: bool = std::env::var("DEBUG_SSE_STREAM") + .unwrap_or("false".to_string()) + .parse::() + .unwrap_or(false); +} pub trait SSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error>; @@ -54,6 +60,9 @@ pub trait SSEParser { // Convert chunk to string and add to buffer let chunk_str = String::from_utf8_lossy(&chunk); + if *DEBUG_SSE_STREAM { + tracing::info!("SSE chunk: {}", chunk_str); + } buffer.push_str(&chunk_str); // Process complete lines from buffer From e047c3b2b1d4d6d76690c49fcc0616740d81d31d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 10 Nov 2025 09:51:30 +0000 Subject: [PATCH 074/105] add require non-empty array --- frontend/src/lib/components/ArgInput.svelte | 14 ++++++++++++++ .../src/lib/components/ArrayTypeNarrowing.svelte | 10 +++++++++- .../src/lib/components/EditableSchemaForm.svelte | 1 + .../lib/components/schema/PropertyEditor.svelte | 3 +++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 3aa15b98f8..b63d3798e5 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -400,6 +400,7 @@ const UUID_PATTERN = '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' const IPV6_PATTERN = '^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$' + function validateInput(pattern: string | undefined, v: any, required: boolean): void { if (nullable && emptyString(v)) { error = '' @@ -407,6 +408,18 @@ } else if (required && (v == undefined || v == null || v === '') && inputCat != 'object') { error = 'Required' valid && (valid = false) + } else if ( + required && + inputCat == 'list' && + extra?.['nonEmpty'] == true && + Array.isArray(v) && + v.length === 0 + ) { + error = 'Required' + valid && (valid = false) + } else if (inputCat == 'list' && !Array.isArray(v)) { + error = 'Expected an array, got ' + typeof v + ' instead' + valid && (valid = false) } else { if (inputCat == 'number' && typeof v === 'number') { let min = extra['min'] @@ -523,6 +536,7 @@ }) $effect(() => { + extra?.['nonEmpty'] let args = [pattern, value, required] as const untrack(() => validateInput(...args)) }) diff --git a/frontend/src/lib/components/ArrayTypeNarrowing.svelte b/frontend/src/lib/components/ArrayTypeNarrowing.svelte index 064faa6950..7feec29713 100644 --- a/frontend/src/lib/components/ArrayTypeNarrowing.svelte +++ b/frontend/src/lib/components/ArrayTypeNarrowing.svelte @@ -24,12 +24,14 @@ properties?: { [name: string]: SchemaProperty } } | undefined + nonEmpty?: boolean | undefined } let { canEditResourceType = false, originalType = undefined, - itemsType = $bindable() + itemsType = $bindable(), + nonEmpty = $bindable() }: Props = $props() let selected: @@ -143,6 +145,12 @@ {/each} + {#if canEditResourceType || originalType == 'string[]' || originalType == 'object[]'}
diff --git a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte index a748d40e96..b22e1ceeac 100644 --- a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte +++ b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte @@ -268,6 +268,7 @@ variant="border" wrapperClasses="p-0 whitespace-nowrap w-fit" btnClasses={twMerge( + 'hover:bg-surface', 'font-mono h-4 py-1 text-2xs', 'font-thin px-1 rounded-[0.275rem]', metaData ? 'rounded-r-none border-r-0.5' : '' From 806a168e185cdaf82356ae1aa0fef77dc1558c8a Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Wed, 12 Nov 2025 13:32:36 +0100 Subject: [PATCH 096/105] Add link to job + update git sync script tip if applicable on fork fail (#7117) * Add link to job + update git sync script tip if applicable on fork fail * Format --- .../workspaceSettings/CreateWorkspace.svelte | 95 +++++++++++++++++-- 1 file changed, 89 insertions(+), 6 deletions(-) diff --git a/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte b/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte index b230658e36..6f6041cc4b 100644 --- a/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte +++ b/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte @@ -4,12 +4,14 @@ import { goto } from '$lib/navigation' import { base } from '$lib/base' import { + JobService, ResourceService, SettingService, UserService, VariableService, WorkspaceService, - type AIProvider + type AIProvider, + type CompletedJob } from '$lib/gen' import { validateUsername } from '$lib/utils' import { logoutWithRedirect } from '$lib/logout' @@ -79,8 +81,41 @@ const WM_FORK_PREFIX = 'wm-fork-' let forkCreationLoading = $state(false) - let forkCreationError = $state("") + let forkCreationError = $state('') let errorMsgs: string[] = $state([]) + let failedSyncJobs: string[] = $state([]) + + async function fetchFailedSyncJobs(jobs: string[]): Promise { + let ret: CompletedJob[] = [] + for (const job of jobs) { + let j = await JobService.getCompletedJob({ + id: job, + workspace: $workspaceStore! + }) + ret.push(j) + } + return ret + } + + function isPathVersionLessThan(path: string | undefined, version: number): boolean { + if (!path || !path.startsWith('hub/')) { + return false + } + + const parts = path.split('/') + + if (parts.length < 2) { + return false + } + + const embeddedVersion = parseInt(parts[1], 10) + + if (isNaN(embeddedVersion)) { + return false + } + + return embeddedVersion < version + } async function createOrForkWorkspace() { const prefixed_id = `${WM_FORK_PREFIX}${id}` @@ -88,7 +123,8 @@ if ($workspaceStore) { forkCreationLoading = true errorMsgs = [] - forkCreationError = "" + failedSyncJobs = [] + forkCreationError = '' let gitSyncJobIds = await WorkspaceService.createWorkspaceForkGitBranch({ workspace: $workspaceStore!, @@ -109,6 +145,7 @@ onProgress: (status) => { if (status.status === 'failure') { errorMsgs.push(status.error ?? 'Deploy fork job failed') + failedSyncJobs.push(jobId) } } }) @@ -123,7 +160,7 @@ return } if (errorMsgs.length != 0) { - forkCreationError = "Failed to create a branch for this fork on the git sync repo(s)" + forkCreationError = 'Failed to create a branch for this fork on the git sync repo(s)' forkCreationLoading = false sendUserToast( `Could not fork workspace ${$workspaceStore} because branch creation failed: ${errorMsgs}`, @@ -143,7 +180,7 @@ }) } catch (e) { forkCreationError = `Failed to create fork '${prefixed_id}'` - errorMsgs.push(e?.body ?? e ?? "Unknown error") + errorMsgs.push(e?.body ?? e ?? 'Unknown error') forkCreationLoading = false sendUserToast(`Could not create fork '${prefixed_id}' ${e}`, true) return @@ -319,11 +356,57 @@ {/if} {#if errorMsgs.length != 0} -
    +
      {#each errorMsgs as errorMsg}
    • - {errorMsg}
    • {/each}
    + {#if failedSyncJobs.length != 0} + More details on the jobs that failed: + {#await fetchFailedSyncJobs(failedSyncJobs)} + + {:then failedJobs} +
      + {#each failedJobs as job} +
    • + - + + {job.id} + +
    • + + {#if isPathVersionLessThan(job.script_path, 28073)} +
      + This job was not running the latest version of the git sync script available on + the hub. You might be able to solve this issue by going to `Workspace Settings` + -> `Git Sync` and updating the script. +
      + {/if} + {/each} +
    + {:catch error} + Tried to fetch jobs to get more information, but failed: {error}. Here are the failed + job ids: +
      + {#each failedSyncJobs as jobId} +
    • + - + + {jobId} + +
    • + {/each} +
    + {/await} + {/if} {/if}