From 612d00367c2a288920c14449561433d199b95cae Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 19 Sep 2025 20:39:27 +0000 Subject: [PATCH 01/28] fix: fix too strict aggrid coldef validation --- .../lib/components/apps/components/display/table/utils.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/components/apps/components/display/table/utils.ts b/frontend/src/lib/components/apps/components/display/table/utils.ts index 3f84b66db5..2cfb668b83 100644 --- a/frontend/src/lib/components/apps/components/display/table/utils.ts +++ b/frontend/src/lib/components/apps/components/display/table/utils.ts @@ -27,9 +27,9 @@ export abstract class AbstractCellRenderer implements ICellRendererComp { eGui: any protected component: | { - refresh: (params: ICellRendererParams) => void - destroy: () => void - } + refresh: (params: ICellRendererParams) => void + destroy: () => void + } | undefined constructor(parentElement = 'span') { // create empty span (or other element) to place svelte component in @@ -275,7 +275,7 @@ export function validateColumnDefs(columnDefs: WindmillColumnDef[]): { } // Check if 'field' property exists and is a non-empty string - if (noField && !(colDef.children && Array.isArray(colDef.children))) { + if (noField && !(colDef.children && Array.isArray(colDef.children)) && !colDef.cellRenderer) { isValid = false errors.push( `Column at index ${index} is missing a valid 'field' property nor having any children.` From 16e35fc3f5da5dc3214bc0f5733d18d745f09805 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 19 Sep 2025 17:36:28 -0400 Subject: [PATCH 02/28] fix ci build (teams) (#6647) --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 694b79ccac..108afd6d32 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -1dc77acb47053b2dd84569d70884456de0188290 +1dc9b8eefe3cd7b0d928e874e2f604f05303ebf3 From 56cdb69e599e295a51284c58d6ff56498ad22fd7 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 00:22:15 +0000 Subject: [PATCH 03/28] fix: add termination handler earlier in lifecycle --- backend/src/main.rs | 31 +++++++++----- backend/windmill-api/src/db.rs | 58 +++++++++++++++++---------- backend/windmill-api/src/lib.rs | 20 +++++---- backend/windmill-worker/src/worker.rs | 11 ++++- 4 files changed, 78 insertions(+), 42 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 87a148d5c3..c3538601dd 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -300,6 +300,19 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { } async fn windmill_main() -> anyhow::Result<()> { + let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); + let mut monitor_killpill_rx = killpill_tx.subscribe(); + let (killpill_phase2_tx, _killpill_phase2_rx) = tokio::sync::broadcast::channel::<()>(2); + let server_killpill_rx = killpill_phase2_tx.subscribe(); + + let shutdown_tx = killpill_tx.clone(); + let shutdown_rx = killpill_tx.subscribe(); + tokio::spawn(async move { + if let Err(e) = windmill_common::shutdown_signal(shutdown_tx, shutdown_rx).await { + tracing::error!("Error in shutdown signal: {e:#}"); + } + }); + dotenv::dotenv().ok(); update_ca_certificates_if_requested(); @@ -463,13 +476,18 @@ async fn windmill_main() -> anyhow::Result<()> { if !skip_migration { // migration code to avoid break - migration_handle = windmill_api::migrate_db(&db).await?; + migration_handle = windmill_api::migrate_db(&db, killpill_rx.resubscribe()).await?; } else { tracing::info!("SKIP_MIGRATION set, skipping db migration...") } } } + if killpill_rx.try_recv().is_ok() { + tracing::info!("Received early killpill, aborting startup"); + return Ok(()); + } + let worker_mode = num_workers > 0; let conn = if mode == Mode::Agent { @@ -483,14 +501,6 @@ async fn windmill_main() -> anyhow::Result<()> { Connection::Sql(db) }; - let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); - let mut monitor_killpill_rx = killpill_tx.subscribe(); - let (killpill_phase2_tx, _killpill_phase2_rx) = tokio::sync::broadcast::channel::<()>(2); - let server_killpill_rx = killpill_phase2_tx.subscribe(); - - let shutdown_signal = - windmill_common::shutdown_signal(killpill_tx.clone(), killpill_tx.subscribe()); - #[cfg(feature = "enterprise")] tracing::info!( " @@ -1257,10 +1267,9 @@ Windmill Community Edition {GIT_VERSION} } if mcp_mode { - futures::try_join!(shutdown_signal, workers_f, server_f)?; + futures::try_join!(workers_f, server_f)?; } else { futures::try_join!( - shutdown_signal, workers_f, monitor_f, server_f, diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index e5d8c433f1..c7ab6feeec 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -101,14 +101,20 @@ impl Migrate for CustomMigrator { let mut r = false; while !r { - r = sqlx::query_scalar!("SELECT pg_try_advisory_lock($1)", lock_id) - .fetch_one(&mut *self.inner) + r = match tokio::time::timeout(std::time::Duration::from_secs(5), sqlx::query_scalar!("SELECT pg_try_advisory_lock($1)", lock_id) + .fetch_one(&mut *self.inner)) .await - .map_err(|e| { - tracing::error!("Error acquiring lock: {e:#}"); - sqlx::migrate::MigrateError::Execute(e) - })? - .unwrap_or(false); + { + Ok(Ok(r)) => r.unwrap_or(false), + Ok(Err(e)) => { + tracing::error!("Error acquiring lock: {e:#}"); + return Err(sqlx::migrate::MigrateError::Execute(e)); + } + Err(e) => { + tracing::error!("Timed out acquiring lock retrying in 5s: {e:#}"); + false + } + }; if !r { tracing::info!("PG migration lock already acquired by another server or worker, a migration is in progress, this may take a long time if you have many jobs and be normal, rechecking in 5s."); tokio::time::sleep(std::time::Duration::from_secs(5)).await; @@ -196,7 +202,10 @@ impl Migrate for CustomMigrator { } } -pub async fn migrate(db: &DB) -> Result>, Error> { +pub async fn migrate( + db: &DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> Result>, Error> { let migrator = db.acquire().await?; let mut custom_migrator = CustomMigrator { inner: migrator }; @@ -211,20 +220,27 @@ pub async fn migrate(db: &DB) -> Result>, Error> { tracing::info!("Could not remove sqlx migrations: {err:#}"); } - match sqlx::migrate!("../migrations") - .run_direct(&mut custom_migrator) - .await - { - Ok(_) => Ok(()), - Err(sqlx::migrate::MigrateError::VersionMissing(e)) => { - tracing::error!("Database had been applied more migrations than this container. - This usually mean than another container on a more recent version migrated the database and this one is on an earlier version. - Please update the container to latest. Not critical, but may cause issues if migration introduced a breaking change. Version missing: {e:#}"); - custom_migrator.unlock().await?; - Ok(()) + tokio::select! { + _ = killpill_rx.recv() => { + tracing::info!("Killpill received, stopping migration"); + return Ok(None); } - Err(err) => Err(err), - }?; + migration_result = sqlx::migrate!("../migrations") + .run_direct(&mut custom_migrator) + => { + match migration_result { + Ok(_) => Ok(()), + Err(sqlx::migrate::MigrateError::VersionMissing(e)) => { + tracing::error!("Database had been applied more migrations than this container. + This usually mean than another container on a more recent version migrated the database and this one is on an earlier version. + Please update the container to latest. Not critical, but may cause issues if migration introduced a breaking change. Version missing: {e:#}"); + custom_migrator.unlock().await?; + Ok(()) + } + Err(err) => Err(err), + }?; + } + } return crate::live_migrations::custom_migrations(&mut custom_migrator, db).await; } diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index eec6574131..3f8f3a993a 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -100,10 +100,10 @@ mod inkeep_oss; mod inputs; mod integration; mod live_migrations; +pub mod openapi; #[cfg(all(feature = "private", feature = "parquet"))] pub mod s3_proxy_ee; mod s3_proxy_oss; -pub mod openapi; mod approvals; #[cfg(all(feature = "enterprise", feature = "private"))] @@ -155,10 +155,10 @@ pub mod stripe_ee; #[cfg(all(feature = "stripe", feature = "enterprise"))] mod stripe_oss; #[cfg(feature = "private")] -pub mod teams_ee; -#[cfg(feature = "private")] pub mod teams_cache_ee; mod teams_cache_oss; +#[cfg(feature = "private")] +pub mod teams_ee; mod teams_oss; mod token; mod tracing_init; @@ -696,9 +696,10 @@ pub async fn run_server( name.map(|x| format!("name={x}")).unwrap_or_default() ); - port_tx - .send(format!("http://localhost:{}", port)) - .expect("Failed to send port"); + if let Err(e) = port_tx.send(format!("http://localhost:{}", port)) { + tracing::error!("Failed to send port: {e:#}"); + return Err(anyhow::anyhow!("Failed to send port, exiting early: {e:#}")); + } let server = server.with_graceful_shutdown(async move { killpill_rx.recv().await.ok(); @@ -794,8 +795,11 @@ async fn openapi_json() -> Response { .unwrap() } -pub async fn migrate_db(db: &DB) -> anyhow::Result>> { - db::migrate(db) +pub async fn migrate_db( + db: &DB, + killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> anyhow::Result>> { + db::migrate(db, killpill_rx) .await .map_err(|e| anyhow::anyhow!("Error migrating db: {e:#}")) } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 2ae1a42f74..06a295f820 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -791,6 +791,7 @@ pub fn start_interactive_worker_shell( loop { if let Ok(_) = killpill_rx.try_recv() { + tracing::info!("Received killpill, exiting worker shell"); break; } else { let pulled_job = match &conn { @@ -2041,8 +2042,14 @@ pub async fn run_worker( } tracing::info!(worker = %worker_name, hostname = %hostname, "waiting for interactive_shell to finish"); if let Some(interactive_shell) = interactive_shell { - if let Err(e) = interactive_shell.await { - tracing::error!("error in awaiting interactive_shell process: {e:?}") + match tokio::time::timeout(Duration::from_secs(10), interactive_shell).await { + Ok(Ok(_)) => {} + Ok(Err(e)) => { + tracing::error!("error in interactive_shell process: {e:?}") + } + Err(_) => { + tracing::error!("timed out awaiting interactive_shell process") + } } } tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name); From fb93024d3e7fac119dac8b595afce0b79efa3b6f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 00:42:43 +0000 Subject: [PATCH 04/28] vacuum skip_locked --- ...1f7f387f5055c47f493271d26731336257384.json | 10 +-- ...b8f4d32815edd2090ccb5d562a89142a7dd9c.json | 12 +++ ...cc6d6bf1df758b30e99bd661da866062ef14f.json | 23 ----- ...3cf7578429009bee4f648e2c1bc3784fdbefc.json | 12 --- ...94f91df7e588d4d2431bc85f4d8734920c8bf.json | 51 ----------- ...c38fc64deb1226aab9dc3bc4465324fce37d1.json | 16 ---- ...ca3761d400391f1f46a8294da3e6c9af63887.json | 15 ---- ...5ef756b8e5c1955fbe111df9ee171dc262338.json | 89 ------------------- ...0cb549a34b96554ae1872355b90304f5dcb76.json | 4 +- ...212a5bd4039b57fab20b163617e33a4c9dd46.json | 14 --- ...77afbd8b3a660b3be27514b517c077c63c238.json | 89 ------------------- backend/Cargo.toml | 4 + backend/update_sqlx.sh | 2 +- backend/windmill-worker/src/worker_utils.rs | 2 +- 14 files changed, 25 insertions(+), 318 deletions(-) create mode 100644 backend/.sqlx/query-0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c.json delete mode 100644 backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json delete mode 100644 backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json delete mode 100644 backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json delete mode 100644 backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json delete mode 100644 backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json delete mode 100644 backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json delete mode 100644 backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json delete mode 100644 backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index d29a18c691..e7ed0aee65 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - true, - true, - true, - true, - true, + false, + false, + false, + false, + false, true, true ] diff --git a/backend/.sqlx/query-0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c.json b/backend/.sqlx/query-0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c.json new file mode 100644 index 0000000000..fbe7e2dec1 --- /dev/null +++ b/backend/.sqlx/query-0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "VACUUM (SKIP_LOCKED) v2_job_queue, v2_job_runtime, v2_job_status, job_perms", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c" +} diff --git a/backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json b/backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json deleted file mode 100644 index 1af42ff529..0000000000 --- a/backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "hash", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f" -} diff --git a/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json b/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json deleted file mode 100644 index 6973159764..0000000000 --- a/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "VACUUM v2_job_queue, v2_job_runtime, v2_job_status, job_perms", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc" -} diff --git a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json deleted file mode 100644 index 6f08d98113..0000000000 --- a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar", - "Varchar", - "Int8", - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - }, - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf" -} diff --git a/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json b/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json deleted file mode 100644 index bce7324fb6..0000000000 --- a/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag) SELECT unnest($1::uuid[]), $2, now(), $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1" -} diff --git a/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json b/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json deleted file mode 100644 index 1afc61978e..0000000000 --- a/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_status (id, flow_status) SELECT unnest($1::uuid[]), $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887" -} diff --git a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json b/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json deleted file mode 100644 index e07a3ccdd7..0000000000 --- a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, raw_flow) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, 1)) RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Int8", - "Varchar", - { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlescriptflow", - "flowscript", - "flownode", - "appscript", - "aiagent" - ] - } - } - }, - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - }, - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Jsonb" - ] - }, - "nullable": [ - false - ] - }, - "hash": "ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338" -} diff --git a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json index 99269c9851..54e94cfb8f 100644 --- a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json +++ b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json @@ -18,8 +18,8 @@ "Left": [] }, "nullable": [ - true, - false + false, + true ] }, "hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76" diff --git a/backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json b/backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json deleted file mode 100644 index a49baeefaf..0000000000 --- a/backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_runtime (id) SELECT unnest($1::uuid[])", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray" - ] - }, - "nullable": [] - }, - "hash": "b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46" -} diff --git a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json b/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json deleted file mode 100644 index 0e42bd0fdb..0000000000 --- a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9 FROM generate_series(1, $10)) RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Int8", - "Varchar", - { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlescriptflow", - "flowscript", - "flownode", - "appscript", - "aiagent" - ] - } - } - }, - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - }, - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238" -} diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ac9af8e0e1..d658d9e880 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -102,6 +102,10 @@ ruby = ["windmill-worker/ruby"] all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java", "ruby"] # For windows we have another set of languages enabled all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] +all_sqlx_features = ["all_languages", "private", "enterprise", "enterprise_saml", "loki", "embedding", "parquet", "prometheus", "flow_testing", + "openidconnect", "cloud", "jemalloc", "tantivy", "sqlx", "kafka", "nats", "otel", "dind", "websocket", "http_trigger", + "postgres_trigger", "mcp", "mqtt_trigger", "sqs_trigger", "gcp_trigger", "smtp", "stripe", + "license", "oauth2", "zip", "static_frontend", "scoped_cache", "agent_worker_server"] [patch.crates-io] object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" } diff --git a/backend/update_sqlx.sh b/backend/update_sqlx.sh index d8bc473f29..159c9e7bd7 100755 --- a/backend/update_sqlx.sh +++ b/backend/update_sqlx.sh @@ -23,7 +23,7 @@ if [[ "$(uname)" == "Darwin" ]]; then sed -i '' 's/^# \(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/\1/' Cargo.toml fi -cargo sqlx prepare --workspace -- --all-targets --all-features +cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features # Undo the samael changes on macOS if [[ "$(uname)" == "Darwin" ]]; then diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index d654e5af65..366cc9cdf7 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -308,7 +308,7 @@ pub(crate) async fn queue_vacuum(conn: &Connection, worker_name: &str, hostname: tokio::task::spawn( (async move { tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue"); - if let Err(e) = sqlx::query!("VACUUM v2_job_queue, v2_job_runtime, v2_job_status, job_perms") + if let Err(e) = sqlx::query!("VACUUM (SKIP_LOCKED) v2_job_queue, v2_job_runtime, v2_job_status, job_perms") .execute(&db2) .await { From e931bdd728bcc8c835b077a5a74e01f31cd86951 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 00:54:29 +0000 Subject: [PATCH 05/28] check --- .github/workflows/backend-check.yml | 12 ++++++------ backend/Cargo.toml | 2 +- backend/update_sqlx.sh | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/backend-check.yml b/.github/workflows/backend-check.yml index 338e1c2008..271d942339 100644 --- a/.github/workflows/backend-check.yml +++ b/.github/workflows/backend-check.yml @@ -20,7 +20,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend timeout-minutes: 16 @@ -41,13 +41,13 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend timeout-minutes: 16 run: | mkdir -p fake_frontend_build - FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features $(./all_features_oss.sh) + FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features all_sqlx_features,private check_ee: runs-on: ubicloud-standard-8 @@ -75,7 +75,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend timeout-minutes: 16 @@ -112,10 +112,10 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check timeout-minutes: 16 working-directory: ./backend run: | mkdir -p fake_frontend_build - FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --all-features + FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features all_sqlx_features,private diff --git a/backend/Cargo.toml b/backend/Cargo.toml index d658d9e880..d7c920cdd4 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -102,7 +102,7 @@ ruby = ["windmill-worker/ruby"] all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java", "ruby"] # For windows we have another set of languages enabled all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] -all_sqlx_features = ["all_languages", "private", "enterprise", "enterprise_saml", "loki", "embedding", "parquet", "prometheus", "flow_testing", +all_sqlx_features = ["all_languages", "enterprise", "enterprise_saml", "loki", "embedding", "parquet", "prometheus", "flow_testing", "openidconnect", "cloud", "jemalloc", "tantivy", "sqlx", "kafka", "nats", "otel", "dind", "websocket", "http_trigger", "postgres_trigger", "mcp", "mqtt_trigger", "sqs_trigger", "gcp_trigger", "smtp", "stripe", "license", "oauth2", "zip", "static_frontend", "scoped_cache", "agent_worker_server"] diff --git a/backend/update_sqlx.sh b/backend/update_sqlx.sh index 159c9e7bd7..77b772c970 100755 --- a/backend/update_sqlx.sh +++ b/backend/update_sqlx.sh @@ -23,7 +23,7 @@ if [[ "$(uname)" == "Darwin" ]]; then sed -i '' 's/^# \(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/\1/' Cargo.toml fi -cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features +cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features,private # Undo the samael changes on macOS if [[ "$(uname)" == "Darwin" ]]; then From 8518b7c897c15ec1d14dc098ea5512ac1e81ef7b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 00:59:03 +0000 Subject: [PATCH 06/28] check --- backend/parsers/windmill-parser-py-imports/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 3b181f1078..bc63d2f8af 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -363,11 +363,11 @@ async fn parse_python_imports_inner( // This way we make sure there is no multiple annotations for same script // and we get detailed span on conflicting versions - #[derive(serde::Serialize, serde::Deserialize)] - struct InlineMetadata { - requires_python: String, - dependencies: Vec, - } + // #[derive(serde::Serialize, serde::Deserialize)] + // struct InlineMetadata { + // requires_python: String, + // dependencies: Vec, + // } let find_requirements = code.lines().find_position(|x| { x.starts_with("#requirements:") From d8577db9e2d646837803501aab4ced1a0b5a7c66 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 01:03:40 +0000 Subject: [PATCH 07/28] check --- backend/windmill-api/src/apps.rs | 16 ++++++++-------- backend/windmill-api/src/workers.rs | 8 ++++---- backend/windmill-api/src/workspaces.rs | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 849ddf877c..d0df61b762 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -147,14 +147,14 @@ fn is_false(b: &bool) -> bool { !b } -#[derive(FromRow, Serialize, Deserialize)] -pub struct AppVersion { - pub id: i64, - pub app_id: Uuid, - pub value: sqlx::types::Json>, - pub created_by: String, - pub created_at: chrono::DateTime, -} +// #[derive(FromRow, Serialize, Deserialize)] +// pub struct AppVersion { +// pub id: i64, +// pub app_id: Uuid, +// pub value: sqlx::types::Json>, +// pub created_by: String, +// pub created_at: chrono::DateTime, +// } #[derive(Debug, Serialize, Deserialize, FromRow)] pub struct AppWithLastVersion { diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index 1165a5e2f3..4e6c5ab086 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -71,10 +71,10 @@ struct WorkerPing { wm_memory_usage: Option, } -#[derive(Serialize, Deserialize)] -struct EnableWorkerQuery { - disable: bool, -} +// #[derive(Serialize, Deserialize)] +// struct EnableWorkerQuery { +// disable: bool, +// } #[derive(Deserialize)] pub struct ListWorkerQuery { diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 1b9d928cf4..e6d3e467fc 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -261,11 +261,11 @@ pub struct WorkspaceSettings { pub auto_add_instance_groups_roles: Option, } -#[derive(sqlx::Type, Serialize, Deserialize, Debug)] -#[sqlx(type_name = "WORKSPACE_KEY_KIND", rename_all = "lowercase")] -pub enum WorkspaceKeyKind { - Cloud, -} +// #[derive(sqlx::Type, Serialize, Deserialize, Debug)] +// #[sqlx(type_name = "WORKSPACE_KEY_KIND", rename_all = "lowercase")] +// pub enum WorkspaceKeyKind { +// Cloud, +// } #[derive(Deserialize)] struct EditCommandScript { @@ -2553,7 +2553,7 @@ async fn clone_variables( source_workspace_id: &str, target_workspace_id: &str, ) -> Result<()> { - sqlx::query!( + sqlx::query!( "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at) SELECT $2, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at FROM variable From f5e23bd8804ba6fbde96cd5f171c6dab48ae81b1 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 01:09:49 +0000 Subject: [PATCH 08/28] check --- backend/ee-repo-ref.txt | 2 +- backend/windmill-api/openapi.yaml | 11 ----------- backend/windmill-api/src/workspaces.rs | 2 +- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 108afd6d32..c102f62212 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -1dc9b8eefe3cd7b0d928e874e2f604f05303ebf3 +c839b0b8eed00d64ce4de78463b23582cab83939 \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c0e3cf4000..cc7940715c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -18318,17 +18318,6 @@ components: - script_path - git_repo_resource_path - UploadFilePart: - type: object - properties: - part_number: - type: integer - tag: - type: string - required: - - part_number - - tag - MetricMetadata: type: object properties: diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index e6d3e467fc..711de8f68d 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -1488,9 +1488,9 @@ async fn delete_git_sync_repository( )) } +#[cfg(feature = "enterprise")] #[derive(Debug, Deserialize)] struct EditDeployUIConfig { - #[cfg(feature = "enterprise")] deploy_ui_settings: Option, } From d2673a8c9fdc10002998ef8b91fe133f7ce8d9b8 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 01:12:51 +0000 Subject: [PATCH 09/28] update rust to 1.90 --- .github/workflows/build_windows_worker_.yml | 2 +- .github/workflows/claude.yml | 2 +- .github/workflows/publish_windows_worker.yml | 2 +- Dockerfile | 2 +- docker/DockerfileFull | 4 ++-- docker/DockerfileFullEe | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index f46d255320..c61f607e17 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -33,7 +33,7 @@ jobs: - name: Setup Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.88.0 + toolchain: 1.90.0 override: true - name: Substitute EE code diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index e5b7ba1b41..39e3de253c 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -82,7 +82,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index a36eca7d85..95545c14a1 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -35,7 +35,7 @@ jobs: - name: Setup Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.88.0 + toolchain: 1.90.0 override: true - name: Substitute EE code diff --git a/Dockerfile b/Dockerfile index e8a807986a..5eb92277bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG DEBIAN_IMAGE=debian:bookworm-slim -ARG RUST_IMAGE=rust:1.88-slim-bookworm +ARG RUST_IMAGE=rust:1.90-slim-bookworm # Build libwindmill_duckdb_ffi_internal.so separately FROM ${RUST_IMAGE} AS windmill_duckdb_ffi_internal_builder diff --git a/docker/DockerfileFull b/docker/DockerfileFull index 1d5f639113..0cf1a83de2 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -1,8 +1,8 @@ FROM ghcr.io/windmill-labs/windmill:dev # Rust -COPY --from=rust:1.88.0 /usr/local/cargo /usr/local/cargo -COPY --from=rust:1.88.0 /usr/local/rustup /usr/local/rustup +COPY --from=rust:1.90.0 /usr/local/cargo /usr/local/cargo +COPY --from=rust:1.90.0 /usr/local/rustup /usr/local/rustup RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 376f994f89..3952113c01 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -20,8 +20,8 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ FROM ghcr.io/windmill-labs/windmill-ee:dev # Rust -COPY --from=rust:1.88.0 /usr/local/cargo /usr/local/cargo -COPY --from=rust:1.88.0 /usr/local/rustup /usr/local/rustup +COPY --from=rust:1.90.0 /usr/local/cargo /usr/local/cargo +COPY --from=rust:1.90.0 /usr/local/rustup /usr/local/rustup RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible From 7a5d3d49e773a6e817ddbda0373f7ee40a692143 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 01:15:30 +0000 Subject: [PATCH 10/28] check --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c102f62212..b088e682a7 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -c839b0b8eed00d64ce4de78463b23582cab83939 \ No newline at end of file +48ffe1df38a1251a50b47ab3def4a3725b449fdd \ No newline at end of file From 0e6f711b6c689f05fdb99911cbd9b7b1784880a7 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 01:16:20 +0000 Subject: [PATCH 11/28] check --- .github/workflows/backend-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend-check.yml b/.github/workflows/backend-check.yml index 271d942339..11f8af5c29 100644 --- a/.github/workflows/backend-check.yml +++ b/.github/workflows/backend-check.yml @@ -47,7 +47,7 @@ jobs: timeout-minutes: 16 run: | mkdir -p fake_frontend_build - FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features all_sqlx_features,private + FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features all_sqlx_features check_ee: runs-on: ubicloud-standard-8 From 2b4bb2be57f50b71f190f097782408404b0a2e1a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 01:23:00 +0000 Subject: [PATCH 12/28] check --- backend/windmill-api/src/apps.rs | 9 --------- backend/windmill-api/src/job_helpers_oss.rs | 9 --------- 2 files changed, 18 deletions(-) diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index d0df61b762..8254e544e7 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -180,15 +180,6 @@ pub struct AppWithLastVersionAndStarred { pub starred: Option, } -#[cfg(feature = "enterprise")] -#[derive(Debug, Serialize, FromRow)] -pub struct AppWithLastVersionAndWorkspace { - #[sqlx(flatten)] - #[serde(flatten)] - pub app: AppWithLastVersion, - pub workspace_id: String, -} - #[derive(Serialize, Deserialize, FromRow)] pub struct AppWithLastVersionAndDraft { #[sqlx(flatten)] diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 90e63f0b76..defb53db76 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -30,15 +30,6 @@ use axum::response::Response; #[cfg(all(feature = "parquet", not(feature = "private")))] use serde::Deserialize; -#[derive(Deserialize)] -#[cfg(not(feature = "private"))] -pub struct LoadImagePreviewQuery { - #[allow(dead_code)] - pub file_key: String, - #[allow(dead_code)] - pub storage: Option, -} - #[derive(Deserialize)] #[cfg(not(feature = "private"))] pub struct DownloadFileQuery { From 93fa61798bf84c9ad0a819a2ed28dae2205b3613 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 01:23:22 +0000 Subject: [PATCH 13/28] check --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b088e682a7..b6e5157568 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -48ffe1df38a1251a50b47ab3def4a3725b449fdd \ No newline at end of file +aa4c7936b99d0de2f7af31cde79dfc7b1b482c29 \ No newline at end of file From 9d8641cdd0ff307ec6fae4dc41e635be992ffc30 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 08:07:41 +0000 Subject: [PATCH 14/28] fix flow jobs --- ...e09f3462c8e39cb3291f0d497742ad4763fa3.json | 19 ------------- ...1db725a7f92d4e43a86bcfa06a4ac7c70ac8f.json | 19 +++++++++++++ ...9d06da7d81e7177a324e3f3f9af1076b3ab66.json | 27 ------------------- ...e4fc6e5804fea077116d339a041c0f2b0c1fa.json | 27 +++++++++++++++++++ ...5518225b1243a9d39ce659b95637c33d0954f.json | 27 ------------------- ...94fafda587749ddd4d53bea5ef36151da5225.json | 27 +++++++++++++++++++ backend/windmill-worker/src/worker_flow.rs | 9 ++++--- 7 files changed, 78 insertions(+), 77 deletions(-) delete mode 100644 backend/.sqlx/query-0088b5bf5cf1c7e47f18c3d05cce09f3462c8e39cb3291f0d497742ad4763fa3.json create mode 100644 backend/.sqlx/query-081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f.json delete mode 100644 backend/.sqlx/query-2d0df658c31d45592dca7add8c69d06da7d81e7177a324e3f3f9af1076b3ab66.json create mode 100644 backend/.sqlx/query-93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa.json delete mode 100644 backend/.sqlx/query-a49e59c814d440bfed7d6bfc2755518225b1243a9d39ce659b95637c33d0954f.json create mode 100644 backend/.sqlx/query-dff20d00a73d7afdda2b3e9867994fafda587749ddd4d53bea5ef36151da5225.json diff --git a/backend/.sqlx/query-0088b5bf5cf1c7e47f18c3d05cce09f3462c8e39cb3291f0d497742ad4763fa3.json b/backend/.sqlx/query-0088b5bf5cf1c7e47f18c3d05cce09f3462c8e39cb3291f0d497742ad4763fa3.json deleted file mode 100644 index 5fa8d88b2e..0000000000 --- a/backend/.sqlx/query-0088b5bf5cf1c7e47f18c3d05cce09f3462c8e39cb3291f0d497742ad4763fa3.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET flow_status = \n CASE WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT],\n $4\n ),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $5),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $6)\n ELSE\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4)\n END\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Jsonb", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "0088b5bf5cf1c7e47f18c3d05cce09f3462c8e39cb3291f0d497742ad4763fa3" -} diff --git a/backend/.sqlx/query-081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f.json b/backend/.sqlx/query-081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f.json new file mode 100644 index 0000000000..75f021b168 --- /dev/null +++ b/backend/.sqlx/query-081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET flow_status = \n CASE WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT],\n $4\n ),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $5),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $6)\n ELSE\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4)\n END\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f" +} diff --git a/backend/.sqlx/query-2d0df658c31d45592dca7add8c69d06da7d81e7177a324e3f3f9af1076b3ab66.json b/backend/.sqlx/query-2d0df658c31d45592dca7add8c69d06da7d81e7177a324e3f3f9af1076b3ab66.json deleted file mode 100644 index 9deb40619a..0000000000 --- a/backend/.sqlx/query-2d0df658c31d45592dca7add8c69d06da7d81e7177a324e3f3f9af1076b3ab66.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET flow_status = \n CASE \n WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n )\n ELSE\n JSONB_SET(JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n ) END\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "2d0df658c31d45592dca7add8c69d06da7d81e7177a324e3f3f9af1076b3ab66" -} diff --git a/backend/.sqlx/query-93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa.json b/backend/.sqlx/query-93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa.json new file mode 100644 index 0000000000..7600acb32a --- /dev/null +++ b/backend/.sqlx/query-93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET flow_status = \n CASE \n WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n ELSE\n JSONB_SET(JSONB_SET(\n flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n END\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa" +} diff --git a/backend/.sqlx/query-a49e59c814d440bfed7d6bfc2755518225b1243a9d39ce659b95637c33d0954f.json b/backend/.sqlx/query-a49e59c814d440bfed7d6bfc2755518225b1243a9d39ce659b95637c33d0954f.json deleted file mode 100644 index 1315df082f..0000000000 --- a/backend/.sqlx/query-a49e59c814d440bfed7d6bfc2755518225b1243a9d39ce659b95637c33d0954f.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET flow_status = \n CASE \n WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n ELSE\n JSONB_SET(JSONB_SET(\n flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n END\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "a49e59c814d440bfed7d6bfc2755518225b1243a9d39ce659b95637c33d0954f" -} diff --git a/backend/.sqlx/query-dff20d00a73d7afdda2b3e9867994fafda587749ddd4d53bea5ef36151da5225.json b/backend/.sqlx/query-dff20d00a73d7afdda2b3e9867994fafda587749ddd4d53bea5ef36151da5225.json new file mode 100644 index 0000000000..2189a90286 --- /dev/null +++ b/backend/.sqlx/query-dff20d00a73d7afdda2b3e9867994fafda587749ddd4d53bea5ef36151da5225.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET flow_status = \n CASE \n WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n )\n ELSE\n JSONB_SET(JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n ) \n END\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "dff20d00a73d7afdda2b3e9867994fafda587749ddd4d53bea5ef36151da5225" +} diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 832af368d1..c5905ef550 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -536,7 +536,7 @@ pub async fn update_flow_status_after_job_completion_internal( sqlx::query_scalar!( "UPDATE v2_job_status SET flow_status = CASE - WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN + WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN JSONB_SET( JSONB_SET(JSONB_SET(JSONB_SET( flow_status, @@ -549,7 +549,8 @@ pub async fn update_flow_status_after_job_completion_internal( JSONB_SET(JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb - ) END + ) + END WHERE id = $2 RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", old_status.step, @@ -611,7 +612,7 @@ pub async fn update_flow_status_after_job_completion_internal( sqlx::query_scalar!( "UPDATE v2_job_status SET flow_status = CASE - WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN + WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN JSONB_SET( JSONB_SET(JSONB_SET(JSONB_SET( flow_status, @@ -1599,7 +1600,7 @@ async fn set_success_and_duration_in_flow_job_success<'c>( if let Some(position) = position { sqlx::query!( "UPDATE v2_job_status SET flow_status = - CASE WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN + CASE WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN JSONB_SET(JSONB_SET(JSONB_SET( flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], From 88b4bc2c9be7bdc77f82b85dd83e3ad1a80ef4df Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 08:11:19 +0000 Subject: [PATCH 15/28] remove tracing err --- backend/windmill-worker/src/worker_flow.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index c5905ef550..bae07fcccb 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -526,12 +526,6 @@ pub async fn update_flow_status_after_job_completion_internal( } else { None }; - tracing::error!( - "flow_job_duration: {:?}, position: {:?} flow: {:?}", - flow_job_duration, - position, - flow - ); let nindex = if let Some(position) = position { sqlx::query_scalar!( "UPDATE v2_job_status SET flow_status = From 9a2110784881a577d44cf17a0715115b8b69a9ed Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 10:17:10 +0000 Subject: [PATCH 16/28] improve job cancelling with new flow jobs locks --- ...a83a2e1f07007b500b556b841181b3adebfb8.json | 18 ++++++++ ...a39101520409c5f241bc8d1b4190bb13260a9.json | 25 +++++++++++ ...94fafda587749ddd4d53bea5ef36151da5225.json | 27 ----------- backend/windmill-worker/src/worker_flow.rs | 45 +++++++++++-------- 4 files changed, 69 insertions(+), 46 deletions(-) create mode 100644 backend/.sqlx/query-054fc3f9d6776b9068816b7568ca83a2e1f07007b500b556b841181b3adebfb8.json create mode 100644 backend/.sqlx/query-d70ddf86eb77a1fdac1e03d2c1aa39101520409c5f241bc8d1b4190bb13260a9.json delete mode 100644 backend/.sqlx/query-dff20d00a73d7afdda2b3e9867994fafda587749ddd4d53bea5ef36151da5225.json diff --git a/backend/.sqlx/query-054fc3f9d6776b9068816b7568ca83a2e1f07007b500b556b841181b3adebfb8.json b/backend/.sqlx/query-054fc3f9d6776b9068816b7568ca83a2e1f07007b500b556b841181b3adebfb8.json new file mode 100644 index 0000000000..461f79a1f2 --- /dev/null +++ b/backend/.sqlx/query-054fc3f9d6776b9068816b7568ca83a2e1f07007b500b556b841181b3adebfb8.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET flow_status = JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $5)\n WHERE id = $2 AND flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "054fc3f9d6776b9068816b7568ca83a2e1f07007b500b556b841181b3adebfb8" +} diff --git a/backend/.sqlx/query-d70ddf86eb77a1fdac1e03d2c1aa39101520409c5f241bc8d1b4190bb13260a9.json b/backend/.sqlx/query-d70ddf86eb77a1fdac1e03d2c1aa39101520409c5f241bc8d1b4190bb13260a9.json new file mode 100644 index 0000000000..91074b72a6 --- /dev/null +++ b/backend/.sqlx/query-d70ddf86eb77a1fdac1e03d2c1aa39101520409c5f241bc8d1b4190bb13260a9.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE v2_job_status SET flow_status = \n JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d70ddf86eb77a1fdac1e03d2c1aa39101520409c5f241bc8d1b4190bb13260a9" +} diff --git a/backend/.sqlx/query-dff20d00a73d7afdda2b3e9867994fafda587749ddd4d53bea5ef36151da5225.json b/backend/.sqlx/query-dff20d00a73d7afdda2b3e9867994fafda587749ddd4d53bea5ef36151da5225.json deleted file mode 100644 index 2189a90286..0000000000 --- a/backend/.sqlx/query-dff20d00a73d7afdda2b3e9867994fafda587749ddd4d53bea5ef36151da5225.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET flow_status = \n CASE \n WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n )\n ELSE\n JSONB_SET(JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n ) \n END\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "dff20d00a73d7afdda2b3e9867994fafda587749ddd4d53bea5ef36151da5225" -} diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index bae07fcccb..0c2da9f761 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -526,33 +526,21 @@ pub async fn update_flow_status_after_job_completion_internal( } else { None }; + let nindex = if let Some(position) = position { sqlx::query_scalar!( - "UPDATE v2_job_status SET flow_status = - CASE - WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN - JSONB_SET( - JSONB_SET(JSONB_SET(JSONB_SET( + " + UPDATE v2_job_status SET flow_status = + JSONB_SET(JSONB_SET( flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), - ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), - ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6), - ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb - ) - ELSE - JSONB_SET(JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), - ARRAY['modules', $1::TEXT, 'iterator', 'index'], - ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb - ) - END + ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb) WHERE id = $2 RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", old_status.step, flow, position as i32, json!(success), - flow_job_duration.as_ref().map(|x| json!(x.started_at)), - flow_job_duration.as_ref().map(|x| json!(x.duration_ms)) ) } else { sqlx::query_scalar!( @@ -573,9 +561,28 @@ pub async fn update_flow_status_after_job_completion_internal( Error::internal_err(format!( "error while fetching iterator index: {e:#}" )) - })? - .ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress")))?; + })?.ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress for flow {flow} at step {}", old_status.step)))?; + if let Some(position) = position { + // if we do this in a single statement, somehow cancelling the flow job will have issues because the RETURNING clause will return None + // Could not figure out why, so we do it in two steps + if let Err(e) = sqlx::query!( + "UPDATE v2_job_status SET flow_status = JSONB_SET(JSONB_SET( + flow_status, + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $4), + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $5) + WHERE id = $2 AND flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL", + old_status.step as i32, + flow, + position as i32, + flow_job_duration.as_ref().map(|x| json!(x.started_at)), + flow_job_duration.as_ref().map(|x| json!(x.duration_ms)) + ) + .execute(&mut *tx) + .await { + tracing::error!("error while updating flow jobs duration: {e:#}"); + } + } // let status_for_debug = sqlx::query!( // "SELECT flow_status FROM v2_job_status WHERE id = $1", // flow From 1afb7a2156570eadb780be00a40eb9395cfb4deb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 11:00:45 +0000 Subject: [PATCH 17/28] improve job cancelling with new flow jobs locks --- ...a83a2e1f07007b500b556b841181b3adebfb8.json | 18 --------- ...e03a6d51c24fc9a2d8ac872aaf41a20af5045.json | 27 ++++++++++++++ ...a39101520409c5f241bc8d1b4190bb13260a9.json | 25 ------------- backend/windmill-worker/src/worker_flow.rs | 37 ++++++------------- .../lib/components/FlowPreviewContent.svelte | 2 +- .../lib/components/FlowStatusViewer.svelte | 8 ++-- .../components/FlowStatusViewerInner.svelte | 2 +- .../lib/components/flows/flowModuleNextId.ts | 1 - frontend/src/lib/components/graph/model.ts | 2 +- 9 files changed, 45 insertions(+), 77 deletions(-) delete mode 100644 backend/.sqlx/query-054fc3f9d6776b9068816b7568ca83a2e1f07007b500b556b841181b3adebfb8.json create mode 100644 backend/.sqlx/query-ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045.json delete mode 100644 backend/.sqlx/query-d70ddf86eb77a1fdac1e03d2c1aa39101520409c5f241bc8d1b4190bb13260a9.json diff --git a/backend/.sqlx/query-054fc3f9d6776b9068816b7568ca83a2e1f07007b500b556b841181b3adebfb8.json b/backend/.sqlx/query-054fc3f9d6776b9068816b7568ca83a2e1f07007b500b556b841181b3adebfb8.json deleted file mode 100644 index 461f79a1f2..0000000000 --- a/backend/.sqlx/query-054fc3f9d6776b9068816b7568ca83a2e1f07007b500b556b841181b3adebfb8.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET flow_status = JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $5)\n WHERE id = $2 AND flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "054fc3f9d6776b9068816b7568ca83a2e1f07007b500b556b841181b3adebfb8" -} diff --git a/backend/.sqlx/query-ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045.json b/backend/.sqlx/query-ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045.json new file mode 100644 index 0000000000..8de90d69d7 --- /dev/null +++ b/backend/.sqlx/query-ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE v2_job_status SET flow_status = \n JSONB_SET(JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6)\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045" +} diff --git a/backend/.sqlx/query-d70ddf86eb77a1fdac1e03d2c1aa39101520409c5f241bc8d1b4190bb13260a9.json b/backend/.sqlx/query-d70ddf86eb77a1fdac1e03d2c1aa39101520409c5f241bc8d1b4190bb13260a9.json deleted file mode 100644 index 91074b72a6..0000000000 --- a/backend/.sqlx/query-d70ddf86eb77a1fdac1e03d2c1aa39101520409c5f241bc8d1b4190bb13260a9.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE v2_job_status SET flow_status = \n JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d70ddf86eb77a1fdac1e03d2c1aa39101520409c5f241bc8d1b4190bb13260a9" -} diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 0c2da9f761..d55ce51be9 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -28,6 +28,7 @@ use serde_json::value::RawValue; use serde_json::{json, Value}; use sqlx::types::Json; use sqlx::{FromRow, Postgres, Transaction}; +use tokio::spawn; use tracing::instrument; use uuid::Uuid; use windmill_common::auth::get_job_perms; @@ -531,16 +532,20 @@ pub async fn update_flow_status_after_job_completion_internal( sqlx::query_scalar!( " UPDATE v2_job_status SET flow_status = - JSONB_SET(JSONB_SET( + JSONB_SET(JSONB_SET(JSONB_SET(JSONB_SET( flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), - ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb) + ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb), + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6) WHERE id = $2 RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", old_status.step, flow, position as i32, json!(success), + flow_job_duration.as_ref().map(|x| json!(x.started_at)).unwrap_or_default(), + flow_job_duration.as_ref().map(|x| json!(x.duration_ms)).unwrap_or_default(), ) } else { sqlx::query_scalar!( @@ -563,26 +568,6 @@ pub async fn update_flow_status_after_job_completion_internal( )) })?.ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress for flow {flow} at step {}", old_status.step)))?; - if let Some(position) = position { - // if we do this in a single statement, somehow cancelling the flow job will have issues because the RETURNING clause will return None - // Could not figure out why, so we do it in two steps - if let Err(e) = sqlx::query!( - "UPDATE v2_job_status SET flow_status = JSONB_SET(JSONB_SET( - flow_status, - ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $4), - ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $5) - WHERE id = $2 AND flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL", - old_status.step as i32, - flow, - position as i32, - flow_job_duration.as_ref().map(|x| json!(x.started_at)), - flow_job_duration.as_ref().map(|x| json!(x.duration_ms)) - ) - .execute(&mut *tx) - .await { - tracing::error!("error while updating flow jobs duration: {e:#}"); - } - } // let status_for_debug = sqlx::query!( // "SELECT flow_status FROM v2_job_status WHERE id = $1", // flow @@ -636,8 +621,8 @@ pub async fn update_flow_status_after_job_completion_internal( flow, position as i32, json!(success), - flow_job_duration.as_ref().map(|x| json!(x.started_at)), - flow_job_duration.as_ref().map(|x| json!(x.duration_ms)) + flow_job_duration.as_ref().map(|x| json!(x.started_at)).unwrap_or_default(), + flow_job_duration.as_ref().map(|x| json!(x.duration_ms)).unwrap_or_default(), ) } else { sqlx::query_scalar!( @@ -1617,8 +1602,8 @@ async fn set_success_and_duration_in_flow_job_success<'c>( flow, position as i32, json!(success), - flow_job_duration.as_ref().map(|x| json!(x.duration_ms)), - flow_job_duration.as_ref().map(|x| json!(x.started_at)) + flow_job_duration.as_ref().map(|x| json!(x.duration_ms)).unwrap_or_default(), + flow_job_duration.as_ref().map(|x| json!(x.started_at)).unwrap_or_default() ) .execute(&mut **tx) .await diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 558a854bca..f2340066e9 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -583,7 +583,7 @@ bind:suspendStatus hideDownloadInGraph={customUi?.downloadLogs === false} wideResults - bind:flowStateStore={flowStateStore.val} + bind:flowState={flowStateStore.val} {jobId} onDone={() => { isRunning = false diff --git a/frontend/src/lib/components/FlowStatusViewer.svelte b/frontend/src/lib/components/FlowStatusViewer.svelte index 6ca2c8fd0f..7d5c1c4a12 100644 --- a/frontend/src/lib/components/FlowStatusViewer.svelte +++ b/frontend/src/lib/components/FlowStatusViewer.svelte @@ -11,7 +11,7 @@ jobId: string initialJob?: Job | undefined workspaceId?: string | undefined - flowStateStore?: FlowState + flowState?: FlowState selectedJobStep?: string | undefined hideFlowResult?: boolean hideTimeline?: boolean @@ -39,7 +39,7 @@ jobId, initialJob = undefined, workspaceId = undefined, - flowStateStore = $bindable({}), + flowState = $bindable({}), selectedJobStep = $bindable(undefined), hideFlowResult = false, hideTimeline = false, @@ -67,7 +67,7 @@ let globalRefreshes: Record Promise)[]> = $state({}) setContext('FlowStatusViewer', { - flowStateStore, + flowState: flowState, suspendStatus, retryStatus, hideDownloadInGraph, @@ -88,7 +88,7 @@ retryStatus.val = {} suspendStatus.val = {} globalRefreshes = {} - flowStateStore.val = {} + flowState.val = {} localDurationStatuses = {} localModuleStates = {} } diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index caf8c7083b..dc101fa520 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -48,7 +48,7 @@ import JobAssetsViewer from './assets/JobAssetsViewer.svelte' let { - flowStateStore, + flowState: flowStateStore, retryStatus, suspendStatus, hideDownloadInGraph, diff --git a/frontend/src/lib/components/flows/flowModuleNextId.ts b/frontend/src/lib/components/flows/flowModuleNextId.ts index 1d0cd14ec6..34bed860f8 100644 --- a/frontend/src/lib/components/flows/flowModuleNextId.ts +++ b/frontend/src/lib/components/flows/flowModuleNextId.ts @@ -6,7 +6,6 @@ import { charsToNumber, numberToChars } from './idUtils' // Computes the next available id export function nextId(flowState: FlowState, fullFlow: OpenFlow): string { const allIds = dfs(fullFlow.value.modules, (fm) => fm.id) - console.log('allIds', allIds) const max = allIds.concat(Object.keys(flowState)).reduce((acc, key) => { if (key.length >= 4) { diff --git a/frontend/src/lib/components/graph/model.ts b/frontend/src/lib/components/graph/model.ts index 8ed90faeeb..990d5cac53 100644 --- a/frontend/src/lib/components/graph/model.ts +++ b/frontend/src/lib/components/graph/model.ts @@ -28,7 +28,7 @@ export type DurationStatus = { } export type FlowStatusViewerContext = { - flowStateStore?: FlowState + flowState?: FlowState retryStatus: StateStore> suspendStatus: StateStore> hideDownloadInGraph?: boolean From a46ccaef90878592a4946678df40a4aa484095e9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 11:08:38 +0000 Subject: [PATCH 18/28] match cancel filter with job kinds --- frontend/src/lib/components/runs/JobsLoader.svelte | 2 +- .../src/routes/(root)/(logged)/runs/[...path]/+page.svelte | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/runs/JobsLoader.svelte b/frontend/src/lib/components/runs/JobsLoader.svelte index 90500ff5f5..920609665a 100644 --- a/frontend/src/lib/components/runs/JobsLoader.svelte +++ b/frontend/src/lib/components/runs/JobsLoader.svelte @@ -70,7 +70,7 @@ jobKindsCat = undefined, minTs = $bindable(undefined), maxTs = $bindable(undefined), - jobKinds = $bindable(''), + jobKinds = $bindable(undefined), queue_count = $bindable(undefined), suspended_count = $bindable(undefined), autoRefresh = true, diff --git a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte index eda1ef6620..ec50dab88c 100644 --- a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte @@ -165,7 +165,7 @@ let queue_count: Tweened | undefined = $state(undefined) let suspended_count: Tweened | undefined = $state(undefined) - let jobKinds: string | undefined = undefined + let jobKinds: string | undefined = $state(undefined) let loading: boolean = $state(false) let paths: string[] = $state([]) let usernames: string[] = $state([]) @@ -820,7 +820,7 @@ computeMinAndMax={manualDatePicker?.computeMinMax} bind:minTs bind:maxTs - {jobKinds} + bind:jobKinds bind:queue_count bind:suspended_count {autoRefresh} From 2eeaf5639dfcdb0114dc5fc323dbd10ef5ceaca5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 11:29:35 +0000 Subject: [PATCH 19/28] feat: force cancel in batch cancel --- backend/windmill-api/openapi.yaml | 4 ++ backend/windmill-api/src/jobs.rs | 18 +++++++- .../(logged)/runs/[...path]/+page.svelte | 42 +++++++++++++++---- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index cc7940715c..dd3dcb5893 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -7727,6 +7727,10 @@ paths: - job parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: force_cancel + in: query + schema: + type: boolean requestBody: description: uuids of the jobs to cancel required: true diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 1b75070217..8089e39fda 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -2055,6 +2055,7 @@ async fn cancel_jobs( db: &DB, username: &str, w_id: &str, + force_cancel: bool, ) -> error::JsonResult> { let mut uuids = vec![]; let mut tx = db.begin().await?; @@ -2114,7 +2115,7 @@ async fn cancel_jobs( w_id, tx, db, - false, + force_cancel, false, ) .await?; @@ -2145,11 +2146,17 @@ async fn cancel_jobs( Ok(Json(uuids)) } +#[derive(Deserialize)] +pub struct CancelSelectionQuery { + force_cancel: Option, +} + async fn cancel_selection( authed: ApiAuthed, Extension(db): Extension, Extension(user_db): Extension, Path(w_id): Path, + Query(query): Query, Json(jobs): Json>, ) -> error::JsonResult> { let mut tx = user_db.begin(&authed).await?; @@ -2163,7 +2170,14 @@ async fn cancel_selection( .await?; tx.commit().await?; - cancel_jobs(jobs_to_cancel, &db, authed.username.as_str(), w_id.as_str()).await + cancel_jobs( + jobs_to_cancel, + &db, + authed.username.as_str(), + w_id.as_str(), + query.force_cancel.unwrap_or(false), + ) + .await } async fn list_filtered_job_uuids( diff --git a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte index ec50dab88c..153d8772ef 100644 --- a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte @@ -186,7 +186,7 @@ confirmBtnText: string loading?: boolean preContent?: string - onConfirm?: () => void + onConfirm?: (forceCancel: boolean) => void type?: ConfirmationModal['$$prop_def']['type'] } = $state(undefined) @@ -544,10 +544,11 @@ } } - async function cancelJobs(uuidsToCancel: string[]) { + async function cancelJobs(uuidsToCancel: string[], forceCancel: boolean = false) { const uuids = await JobService.cancelSelection({ workspace: $workspaceStore ?? '', - requestBody: uuidsToCancel + requestBody: uuidsToCancel, + forceCancel: forceCancel }) selectedIds = [] jobsLoader?.loadJobs(minTs, maxTs, true, true) @@ -556,6 +557,7 @@ } async function onCancelFilteredJobs() { + forceCancelInPopup = false askingForConfirmation = { title: 'Confirm cancelling all jobs corresponding to the selected filters', confirmBtnText: 'Loading...', @@ -570,18 +572,19 @@ title: `Confirm cancelling all jobs corresponding to the selected filters (${jobIdsToCancel.length} jobs)`, confirmBtnText: `Cancel ${jobIdsToCancel.length} jobs that matched the filters`, preContent: selectedFiltersString, - onConfirm: () => { - cancelJobs(jobIdsToCancel) + onConfirm: (forceCancel) => { + cancelJobs(jobIdsToCancel, forceCancel) } } } async function onCancelSelectedJobs() { + forceCancelInPopup = true askingForConfirmation = { confirmBtnText: `Cancel ${selectedIds.length} jobs`, title: 'Confirm cancelling the selected jobs', - onConfirm: () => { - cancelJobs(selectedIds) + onConfirm: (forceCancel) => { + cancelJobs(selectedIds, forceCancel) } } } @@ -798,6 +801,8 @@ const smallScreenWidth = 1920 const verySmallScreenWidth = 1300 + + let forceCancelInPopup = $state(false) { const func = askingForConfirmation?.onConfirm - await func?.() + await func?.(forceCancelInPopup) askingForConfirmation = undefined }} type={askingForConfirmation?.type} @@ -853,6 +858,27 @@ > {#if askingForConfirmation?.preContent} 
{askingForConfirmation.preContent}
+ + {#if forceCancelInPopup} +
+

+ Force cancel is enabled. This is dangerous, only do this if you have no alternatives. + Instead of being gracefully cancelled, all jobs will be immediately sent to the completed + job table regardless of them being processed or not or part of running flows. You may end + up in an inconsistent state. +

+
+ {/if} {/if} From 194887e97b3250ef28b0ddf5c03932148858ca24 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 20 Sep 2025 12:50:15 +0000 Subject: [PATCH 20/28] fix: allow variable picker in password field --- frontend/src/lib/components/ArgInput.svelte | 30 ++++----- frontend/src/lib/components/Password.svelte | 33 ++++++---- .../lib/components/PasswordArgInput.svelte | 61 ++++++++++++++----- .../schedules/ScheduleEditorInner.svelte | 1 + 4 files changed, 84 insertions(+), 41 deletions(-) diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 71bf07a8f5..711e615e09 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -540,7 +540,7 @@
{/key} - {#if !disabled && itemPicker && extra?.['disableVariablePicker'] != true} - - - {/if} + {/if} + {#if !disabled && itemPicker && extra?.['disableVariablePicker'] != true} + + {/if}
{@render variableInput()} diff --git a/frontend/src/lib/components/Password.svelte b/frontend/src/lib/components/Password.svelte index 2d031da318..a490cef6df 100644 --- a/frontend/src/lib/components/Password.svelte +++ b/frontend/src/lib/components/Password.svelte @@ -1,15 +1,26 @@ @@ -29,7 +40,7 @@ : ''} text-sm h-9" type="password" bind:value={password} - on:keydown + onkeydown={bubble('keydown')} autocomplete="new-password" {placeholder} {disabled} @@ -41,7 +52,7 @@ : ''} text-sm h-9" type="text" bind:value={password} - on:keydown + onkeydown={bubble('keydown')} autocomplete="new-password" {placeholder} {disabled} diff --git a/frontend/src/lib/components/PasswordArgInput.svelte b/frontend/src/lib/components/PasswordArgInput.svelte index e4b349b531..200d15d49a 100644 --- a/frontend/src/lib/components/PasswordArgInput.svelte +++ b/frontend/src/lib/components/PasswordArgInput.svelte @@ -2,24 +2,32 @@ import { VariableService } from '$lib/gen' import { userStore, workspaceStore } from '$lib/stores' import { generateRandomString } from '$lib/utils' + import { Button } from './common' import Password from './Password.svelte' + import { untrack } from 'svelte' - export let value: string | undefined = undefined - export let disabled: boolean + interface Props { + value?: string | undefined + disabled: boolean + } - let path = '' - let password = value && typeof value === 'string' && !value.startsWith('$var:') ? value : '' + let { value = $bindable(undefined), disabled }: Props = $props() + + let path = $state('') + let password = $state( + value && typeof value === 'string' && !value.startsWith('$var:') ? value : '' + ) let isGenerating = false + + let userPrefix = $derived( + 'u/' + ($userStore?.username ?? $userStore?.email)?.split('@')[0] + '/secret_arg/' + ) async function generateValue() { if (isGenerating) return isGenerating = true try { - let npath = - 'u/' + - ($userStore?.username ?? $userStore?.email)?.split('@')[0] + - '/secret_arg/' + - generateRandomString(12) + let npath = userPrefix + generateRandomString(12) let nvalue = '$var:' + npath await VariableService.createVariable({ workspace: $workspaceStore!, @@ -32,6 +40,7 @@ } }) path = npath + console.log('generated', nvalue) value = nvalue debouncedUpdate() } finally { @@ -59,13 +68,33 @@ timeout = setTimeout(updateValue, 500) } - $: password && debouncedUpdate() + $effect(() => { + password && untrack(() => debouncedUpdate()) + }) - $: $workspaceStore && - ($userStore?.username || $userStore?.email) && - path == '' && - password != '' && - generateValue() + $effect(() => { + $workspaceStore && + ($userStore?.username || $userStore?.email) && + path == '' && + password != '' && + untrack(() => generateValue()) + }) - +{#if value?.startsWith('$var:') && !value.startsWith('$var:' + userPrefix)} +
+ Linked to static variable + +
+{:else} + +{/if} diff --git a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte index 7ad0ab7d80..e3309f99e0 100644 --- a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte @@ -858,6 +858,7 @@ {:then Module} Date: Sat, 20 Sep 2025 13:41:15 +0000 Subject: [PATCH 21/28] fix: retry python relative imports on errno 104 --- backend/windmill-worker/loader.py | 45 ++++++++++++--------- frontend/src/lib/components/ArgInput.svelte | 2 +- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/backend/windmill-worker/loader.py b/backend/windmill-worker/loader.py index 1361e06ead..c387e5e0fc 100644 --- a/backend/windmill-worker/loader.py +++ b/backend/windmill-worker/loader.py @@ -51,25 +51,32 @@ class WindmillFinder(MetaPathFinder): url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py{query_params}" req = urllib.request.Request(url, None, headers) - try: - req_start = time.time() - with urllib.request.urlopen(req) as response: - os.makedirs(folder, exist_ok=True) - r = response.read().decode("utf-8") - if r == "WINDMILL_IS_FOLDER": - return ModuleSpec(name, WindmillLoader(name)) - with open(fullpath, "w+") as f: - f.write(r) - return ModuleSpec(name, SourceFileLoader(name, fullpath)) - except urllib.error.HTTPError as e: - duration = time.time() - req_start - if e.code != 404: - print(f"Error fetching script {script_path}: HTTP {e.code} - {e.reason} - {duration}s") - return ModuleSpec(name, WindmillLoader(name)) - except Exception as e: - duration = time.time() - req_start - print(f"Error fetching script {script_path}: {e} - {duration}s") - return ModuleSpec(name, WindmillLoader(name)) + + for attempt in range(4): # 0, 1, 2, 3 = up to 3 retries + try: + req_start = time.time() + with urllib.request.urlopen(req) as response: + os.makedirs(folder, exist_ok=True) + r = response.read().decode("utf-8") + if r == "WINDMILL_IS_FOLDER": + return ModuleSpec(name, WindmillLoader(name)) + with open(fullpath, "w+") as f: + f.write(r) + return ModuleSpec(name, SourceFileLoader(name, fullpath)) + except urllib.error.HTTPError as e: + duration = time.time() - req_start + if e.code != 404: + print(f"Error fetching script {script_path}: HTTP {e.code} - {e.reason} - {duration}s") + return ModuleSpec(name, WindmillLoader(name)) + except Exception as e: + duration = time.time() - req_start + # Check if this is errno 104 (Connection reset by peer) and we have retries left + if (hasattr(e, 'errno') and e.errno == 104) and attempt < 3: + print(f"Connection reset (errno 104) fetching script {script_path}, retrying in 3s (attempt {attempt + 1}/3)") + time.sleep(3) + continue + print(f"Error fetching script {script_path}: {e} - {duration}s") + return ModuleSpec(name, WindmillLoader(name)) diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 711e615e09..6df54a8054 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -1338,7 +1338,7 @@