From da95588b253e8bb2a06b9792674b4d691384f55a Mon Sep 17 00:00:00 2001 From: hugocasa Date: Sat, 2 May 2026 15:12:43 +0200 Subject: [PATCH 01/56] fix(forks): strip mode/enabled from merge-UI deploy payload (#9008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(forks): strip mode/enabled from merge-UI deploy payload The CompareWorkspaces merge UI deploys a trigger by fetching the full GET response from the source workspace and spreading it into an updateXTrigger call on the target. That spread includes `mode` (and the legacy `enabled`), so a fork→parent (or parent→fork) deploy would overwrite the target's enabled/disabled state — silently disabling a parent's trigger when its config is merged from a freshly-cloned fork (clones are forced `mode='disabled'`). Affects azure/email/gcp/http whose `update_trigger` SQL writes the `mode` column. Strip `mode`/`enabled` in `getTriggersDeployData` so the backend's existing `is_mode_unspecified()` preservation in `update_trigger` keeps the target row's `mode` untouched. The same preservation already protects the YAML/CLI round-trip (where the tarball export strips these fields); this extends the same guarantee to the merge-UI path. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(forks): wire Azure and Email triggers through merge UI deploy CompareWorkspaces.svelte already lists Azure and Email triggers in its diff (`triggerServices`), but `getTriggersDeployData` and `existsTrigger` were missing the corresponding branches. Deploying either kind from the merge UI threw "Unexpected trigger kind". Add the branches with the same `stripOperationalState` pattern as the rest, and extend the `triggersKind` whitelist in `checkItemExists`. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- frontend/src/lib/utils_deployable.ts | 73 +++++++++++++++++++--- frontend/src/lib/utils_workspace_deploy.ts | 4 +- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/frontend/src/lib/utils_deployable.ts b/frontend/src/lib/utils_deployable.ts index 24341e323f..efe8b4faab 100644 --- a/frontend/src/lib/utils_deployable.ts +++ b/frontend/src/lib/utils_deployable.ts @@ -1,5 +1,7 @@ import { minimatch } from 'minimatch' import { + AzureTriggerService, + EmailTriggerService, GcpTriggerService, HttpTriggerService, KafkaTriggerService, @@ -84,15 +86,36 @@ export async function existsTrigger( return await WebsocketTriggerService.existsWebsocketTrigger(data) } else if (triggerKind === 'nats') { return await NatsTriggerService.existsNatsTrigger(data) + } else if (triggerKind === 'azure') { + return await AzureTriggerService.existsAzureTrigger(data) + } else if (triggerKind === 'emails') { + return await EmailTriggerService.existsEmailTrigger(data) } else if (triggerKind === 'schedules') { return await ScheduleService.existsSchedule(data) } throw new Error( - `Unexpected trigger kind ${triggerKind}. Allowed kinds are: routes, kafka, mqtt, postgres, sqs, gcp, websockets, nats, schedules.` + `Unexpected trigger kind ${triggerKind}. Allowed kinds are: routes, kafka, mqtt, postgres, sqs, gcp, websockets, nats, azure, emails, schedules.` ) } +/** + * Strip operational state (`mode`, `enabled`) from a trigger/schedule payload + * before sending it to an update endpoint via the merge UI. The backend's + * `update_trigger` handler preserves the target row's existing `mode` when + * both fields are absent from the request (`is_mode_unspecified()`), so + * stripping here lets a fork→parent (or parent→fork) deploy carry config + * changes without flipping the target's enabled/disabled state. Schedules' + * `EditSchedule` already lacks `enabled` on the backend, but stripping keeps + * the intent explicit and matches the YAML/CLI round-trip behavior. + */ +function stripOperationalState>( + payload: T +): Omit { + const { mode: _mode, enabled: _enabled, ...rest } = payload + return rest +} + /** * Get trigger deployment data with optional permissioned_as preservation. * @param onBehalfOf - If set, the trigger will be deployed with this permissioned_as (u/username or g/group) and preserve_permissioned_as=true. @@ -113,7 +136,7 @@ export async function getTriggersDeployData( return { data: { - ...sqsTrigger, + ...stripOperationalState(sqsTrigger), permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs }, @@ -128,7 +151,7 @@ export async function getTriggersDeployData( return { data: { - ...kafkaTrigger, + ...stripOperationalState(kafkaTrigger), permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs }, @@ -143,7 +166,7 @@ export async function getTriggersDeployData( return { data: { - ...mqttTrigger, + ...stripOperationalState(mqttTrigger), permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs }, @@ -158,7 +181,7 @@ export async function getTriggersDeployData( return { data: { - ...natsTrigger, + ...stripOperationalState(natsTrigger), permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs }, @@ -179,7 +202,7 @@ export async function getTriggersDeployData( } const data: GcpTriggerData = { - ...gcpTrigger, + ...stripOperationalState(gcpTrigger), delivery_config: gcpTrigger.delivery_config ?? undefined, base_endpoint: gcpTrigger.delivery_type === 'push' ? `${window.location.origin}${base}` : undefined, @@ -200,7 +223,7 @@ export async function getTriggersDeployData( return { data: { - ...postgresTrigger, + ...stripOperationalState(postgresTrigger), permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs }, @@ -215,7 +238,7 @@ export async function getTriggersDeployData( return { data: { - ...websocketTrigger, + ...stripOperationalState(websocketTrigger), permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs }, @@ -230,13 +253,43 @@ export async function getTriggersDeployData( return { data: { - ...httpTrigger, + ...stripOperationalState(httpTrigger), permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs }, createFn: HttpTriggerService.createHttpTrigger, updateFn: HttpTriggerService.updateHttpTrigger } + } else if (kind === 'azure') { + const azureTrigger = await AzureTriggerService.getAzureTrigger({ + workspace: workspace!, + path: path + }) + + return { + data: { + ...stripOperationalState(azureTrigger), + permissioned_as: onBehalfOf, + preserve_permissioned_as: preservePermissionedAs + }, + createFn: AzureTriggerService.createAzureTrigger, + updateFn: AzureTriggerService.updateAzureTrigger + } + } else if (kind === 'emails') { + const emailTrigger = await EmailTriggerService.getEmailTrigger({ + workspace: workspace!, + path: path + }) + + return { + data: { + ...stripOperationalState(emailTrigger), + permissioned_as: onBehalfOf, + preserve_permissioned_as: preservePermissionedAs + }, + createFn: EmailTriggerService.createEmailTrigger, + updateFn: EmailTriggerService.updateEmailTrigger + } } else if (kind === 'schedules') { const schedulesTrigger = await ScheduleService.getSchedule({ workspace: workspace!, @@ -244,7 +297,7 @@ export async function getTriggersDeployData( }) return { data: { - ...schedulesTrigger, + ...stripOperationalState(schedulesTrigger), // permissioned_as is only set on create, not update permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs diff --git a/frontend/src/lib/utils_workspace_deploy.ts b/frontend/src/lib/utils_workspace_deploy.ts index 7ff09c8d0d..9ddf68520f 100644 --- a/frontend/src/lib/utils_workspace_deploy.ts +++ b/frontend/src/lib/utils_workspace_deploy.ts @@ -174,7 +174,9 @@ export async function checkItemExists( 'schedules', 'sqs', 'websockets', - 'gcp' + 'gcp', + 'azure', + 'emails' ] if ( additionalInformation?.triggers && From 392888d11311d570dc55f30efdee989f866582b0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 3 May 2026 07:14:14 +0000 Subject: [PATCH 02/56] docs: add SAFETY comments to all dynamic SQL call sites (#9009) * docs: add SAFETY comments to all dynamic SQL call sites Co-Authored-By: Claude Opus 4.5 * docs: address review feedback on SAFETY comments - Fix missed comment for obo_triggers loop in offboarding.rs - Fix variable name in comment (table -> table_name) in offboarding.rs - Fix api-settings comment to reference inline VALID_NAME regex, not validate_dbname() - Add SAFETY comments to batch_execute calls in api-settings - Fix db.rs comment: PG_SCHEMA is env var, not compile-time constant - Add doc comments on RunnableSettingsTraitInternal constants * docs: remove misleading SAFETY comment on static SQL --------- Co-authored-by: Claude Opus 4.5 --- backend/windmill-api-groups/src/granular_acls.rs | 3 +++ backend/windmill-api-settings/src/lib.rs | 2 ++ backend/windmill-api-users/src/users.rs | 1 + backend/windmill-api/src/offboarding.rs | 4 ++++ backend/windmill-api/src/trash.rs | 2 ++ backend/windmill-common/src/db.rs | 1 + backend/windmill-common/src/lib.rs | 3 +++ backend/windmill-common/src/runnable_settings/mod.rs | 5 +++++ backend/windmill-trigger-http/src/handler.rs | 1 + backend/windmill-trigger/src/handler.rs | 10 ++++++++++ backend/windmill-trigger/src/listener.rs | 4 ++++ 11 files changed, 36 insertions(+) diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index eac532dc78..95243b6626 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -131,6 +131,7 @@ async fn add_granular_acl( } } + // SAFETY: `kind` has been validated against the `KINDS` allowlist before reaching this function. let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!( "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2), \ true) WHERE {identifier} = $3 AND workspace_id = $4 RETURNING extra_perms" @@ -294,6 +295,7 @@ async fn remove_granular_acl( require_owner_of_path(&authed, path)?; } + // SAFETY: `kind` has been validated against the `KINDS` allowlist before reaching this function. let obj_o = sqlx::query_scalar::<_, bool>(&format!( "WITH old AS ( SELECT extra_perms->$1 as old_write FROM {kind} @@ -419,6 +421,7 @@ async fn get_granular_acls( } else { "path" }; + // SAFETY: `kind` has been validated against the `KINDS` allowlist before reaching this function. let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!( "SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2" )) diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 2251246f5d..f027046808 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -1132,6 +1132,7 @@ async fn setup_custom_instance_pg_database_inner( logs.created_database = "SKIP".to_string(); if !db_exists { + // SAFETY: `dbname` has been validated by the VALID_NAME regex and length checks above (lines 1088–1120). sqlx::query(&format!("CREATE DATABASE \"{dbname}\"")) .execute(db) .await?; @@ -1144,6 +1145,7 @@ async fn setup_custom_instance_pg_database_inner( logs.db_connect = "OK".to_string(); + // SAFETY: `dbname` has been validated by the VALID_NAME regex and length checks above. client .batch_execute(&format!( "GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user; diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 951060b531..f664ab09b8 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -1698,6 +1698,7 @@ pub async fn delete_workspace_user_internal( "azure_trigger", "email_trigger", ]; + // SAFETY: `table` comes from a hardcoded allowlist `extra_perms_tables`, not user input. for table in &extra_perms_tables { sqlx::query(&format!( "UPDATE {table} SET extra_perms = extra_perms - ('u/' || $1) \ diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index a5d15165a2..c929a85a84 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -194,6 +194,7 @@ async fn get_offboard_preview( ]; let mut triggers = HashMap::new(); for table in &trigger_tables { + // SAFETY: `table` comes from a hardcoded allowlist `trigger_tables`, not user input. let paths: Vec = sqlx::query_scalar(&format!( "SELECT path FROM {table} WHERE path LIKE $1 AND workspace_id = $2" )) @@ -246,6 +247,7 @@ async fn get_offboard_preview( ).fetch_all(db).await?; let mut obo_triggers = HashMap::new(); + // SAFETY: `table` comes from a hardcoded allowlist `trigger_tables`, not user input. for table in &trigger_tables { let paths: Vec = sqlx::query_scalar(&format!( "SELECT path FROM {table} WHERE permissioned_as = $1 AND NOT path LIKE $2 AND workspace_id = $3" @@ -759,6 +761,7 @@ async fn check_path_conflicts( "flow" => " AND NOT t1.archived", _ => "", }; + // SAFETY: `table_name` comes from a hardcoded allowlist `tables`, not user input. let rows: Vec = sqlx::query_scalar(&format!( "SELECT REGEXP_REPLACE(t1.path, '^u/' || $1 || '/', $3) \ FROM {table} t1 \ @@ -1027,6 +1030,7 @@ async fn offboard_user_from_workspace<'c>( ]; let mut triggers_reassigned: i64 = 0; + // SAFETY: `table` comes from a hardcoded allowlist `trigger_tables`, not user input. for table in &trigger_tables { let count: i64 = sqlx::query_scalar(&format!( "WITH updated AS ( \ diff --git a/backend/windmill-api/src/trash.rs b/backend/windmill-api/src/trash.rs index 83e607a7dc..d0d7ddc6c2 100644 --- a/backend/windmill-api/src/trash.rs +++ b/backend/windmill-api/src/trash.rs @@ -480,6 +480,7 @@ async fn restore_trigger(tx: &mut sqlx::PgConnection, item: &TrashItemWithData) ))); } + // SAFETY: `table_name` has been validated against the `valid_tables` allowlist above. let exists: bool = sqlx::query_scalar(&format!( "SELECT EXISTS(SELECT 1 FROM {} WHERE path = $1 AND workspace_id = $2)", table_name @@ -500,6 +501,7 @@ async fn restore_trigger(tx: &mut sqlx::PgConnection, item: &TrashItemWithData) .get("row") .ok_or_else(|| Error::internal_err("Invalid trash data for trigger"))?; + // SAFETY: `table_name` has been validated against the `valid_tables` allowlist above. sqlx::query(&format!( "INSERT INTO {} SELECT * FROM jsonb_populate_record(null::{}, $1)", table_name, table_name diff --git a/backend/windmill-common/src/db.rs b/backend/windmill-common/src/db.rs index e122c8453e..0048abf438 100644 --- a/backend/windmill-common/src/db.rs +++ b/backend/windmill-common/src/db.rs @@ -233,6 +233,7 @@ impl UserDB { let mut tx = self.db.begin().await?; if let Some(schema) = PG_SCHEMA.as_ref() { + // SAFETY: `schema` is an operator-controlled environment variable (PG_SCHEMA), set at deploy time and never user-supplied. sqlx::query(&format!("SET LOCAL search_path TO {}", schema)) .execute(&mut *tx) .await?; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index ef0b2dc5bf..7c7e3f79ed 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -778,6 +778,7 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu if db_exists { // Terminate active connections + // SAFETY: `dbname` has been validated via validate_dbname() before reaching this point. if let Err(e) = sqlx::query(&format!( "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{}' AND pid <> pg_backend_pid()", dbname.replace('\'', "''") @@ -789,6 +790,7 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu } // Drop the database + // SAFETY: `dbname` has been validated via validate_dbname() before reaching this point. sqlx::query(&format!("DROP DATABASE IF EXISTS \"{}\"", dbname)) .execute(db) .await @@ -837,6 +839,7 @@ pub async fn create_custom_instance_database( ))); } + // SAFETY: `dbname` has been validated via validate_dbname() before reaching this point. sqlx::query(&format!("CREATE DATABASE \"{}\"", dbname)) .execute(db) .await diff --git a/backend/windmill-common/src/runnable_settings/mod.rs b/backend/windmill-common/src/runnable_settings/mod.rs index 09bca46bbc..c5692d40bd 100644 --- a/backend/windmill-common/src/runnable_settings/mod.rs +++ b/backend/windmill-common/src/runnable_settings/mod.rs @@ -62,6 +62,7 @@ pub trait RunnableSettingsTrait: async move { let v = RUNNABLE_INDIVIDUAL_SETTINGS .get_or_insert_async(hash, async { + // SAFETY: INCLUDE_FIELDS and SETTINGS_NAME are compile-time constants, not user input. let r = sqlx::query_as::(&format!( "SELECT {} FROM {} WHERE hash = $1", Self::INCLUDE_FIELDS.iter().join(","), @@ -138,7 +139,11 @@ mod private_mod { pub type Q<'a> = Query<'a, Postgres, ::Arguments<'a>>; pub trait RunnableSettingsTraitInternal { + /// Table name used in dynamic SQL via `format!()`. This is a compile-time + /// constant set by each settings impl — it is never user-controllable. const SETTINGS_NAME: &'static str; + /// Column list used in dynamic SQL SELECT via `format!()`. This is a + /// compile-time constant — never user-controllable. const INCLUDE_FIELDS: &'static [&'static str]; fn bind_arguments<'a>(&'a self, q: Q<'a>) -> Q<'a>; diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs index 9aaad6e7c8..74fd748f55 100644 --- a/backend/windmill-trigger-http/src/handler.rs +++ b/backend/windmill-trigger-http/src/handler.rs @@ -628,6 +628,7 @@ impl TriggerCrud for HttpTrigger { workspace_id: &str, path: &str, ) -> Result { + // SAFETY: Self::TABLE_NAME is a compile-time constant, not user input. let deleted = sqlx::query(&format!( "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", Self::TABLE_NAME diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 855961cf5e..f425f97b53 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -52,6 +52,8 @@ pub trait TriggerCrud: Send + Sync + 'static { type TriggerConfigRequest: Debug + DeserializeOwned + Serialize + Send + Sync; type TestConnectionConfig: Debug + DeserializeOwned + Serialize + Send + Sync; + /// Table name used in dynamic SQL queries via `format!()`. This is a compile-time + /// constant set by each trigger impl — it is never user-controllable. const TABLE_NAME: &'static str; const TRIGGER_TYPE: &'static str; const SUPPORTS_SERVER_STATE: bool; @@ -189,6 +191,7 @@ pub trait TriggerCrud: Send + Sync + 'static { } async fn exists(&self, db: &DB, workspace_id: &str, path: &str) -> Result { + // SAFETY: Self::TABLE_NAME is a compile-time constant, not user input. let exists = sqlx::query_scalar(&format!( "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", Self::TABLE_NAME @@ -207,6 +210,7 @@ pub trait TriggerCrud: Send + Sync + 'static { workspace_id: &str, path: &str, ) -> Result { + // SAFETY: Self::TABLE_NAME is a compile-time constant, not user input. let deleted = sqlx::query(&format!( "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", Self::TABLE_NAME @@ -234,6 +238,7 @@ pub trait TriggerCrud: Send + Sync + 'static { ) -> Result { let permissioned_as = windmill_common::users::username_to_permissioned_as(&authed.username); let updated = if Self::SUPPORTS_SERVER_STATE { + // SAFETY: Self::TABLE_NAME is a compile-time constant. sqlx::query(&format!( r#" UPDATE @@ -260,6 +265,7 @@ pub trait TriggerCrud: Send + Sync + 'static { .await? .rows_affected() } else { + // SAFETY: Self::TABLE_NAME is a compile-time constant. sqlx::query(&format!( r#" UPDATE @@ -298,6 +304,7 @@ pub trait TriggerCrud: Send + Sync + 'static { is_flow: bool, script_path: &str, ) -> i64 { + // SAFETY: Self::TABLE_NAME is a compile-time constant. let count = sqlx::query_scalar(&format!( r#" SELECT @@ -466,6 +473,7 @@ async fn create_trigger( .await?; if let Some(ref labels) = labels { + // SAFETY: T::TABLE_NAME is a compile-time constant. sqlx::query(&format!( "UPDATE {} SET labels = $1 WHERE workspace_id = $2 AND path = $3", T::TABLE_NAME @@ -615,6 +623,7 @@ async fn update_trigger( .await?; if let Some(ref labels) = labels { + // SAFETY: T::TABLE_NAME is a compile-time constant. sqlx::query(&format!( "UPDATE {} SET labels = $1 WHERE workspace_id = $2 AND path = $3", T::TABLE_NAME @@ -692,6 +701,7 @@ async fn delete_trigger( let mut tx = user_db.begin(&authed).await?; // Capture trigger data for trashbin before deleting + // SAFETY: T::TABLE_NAME is a compile-time constant. let trash_data: Option = sqlx::query_scalar(&format!( "SELECT jsonb_build_object('row', to_jsonb(t), 'table_name', '{table}') FROM {table} t WHERE path = $1 AND workspace_id = $2", table = T::TABLE_NAME diff --git a/backend/windmill-trigger/src/listener.rs b/backend/windmill-trigger/src/listener.rs index 8344b71867..e327ed3155 100644 --- a/backend/windmill-trigger/src/listener.rs +++ b/backend/windmill-trigger/src/listener.rs @@ -234,6 +234,7 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { listening_trigger: &ListeningTrigger, error: Option<&str>, ) -> Option<()> { + // SAFETY: Self::TABLE_NAME is a compile-time constant, not user input. let updated = sqlx::query_scalar::<_, i32>(&format!( r#" UPDATE @@ -336,6 +337,7 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { listening_trigger: &ListeningTrigger, ) { if listening_trigger.trigger_mode { + // SAFETY: Self::TABLE_NAME is a compile-time constant. let _ = sqlx::query(&format!( r#" UPDATE @@ -384,6 +386,7 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { error: String, ) { if listening_trigger.trigger_mode { + // SAFETY: Self::TABLE_NAME is a compile-time constant. let report_status = sqlx::query(&format!( r#" UPDATE @@ -660,6 +663,7 @@ pub async fn listen_to_unlistened_events( Ok(mut unlistend_enabled_triggers) => { unlistend_enabled_triggers.shuffle(&mut rand::rng()); for trigger in unlistend_enabled_triggers { + // SAFETY: T::TABLE_NAME is a compile-time constant. let has_lock = sqlx::query_scalar(&format!( r#" UPDATE From 1174d7d77fe7a44569d21993295936cbbf91e3db Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 4 May 2026 08:39:45 +0000 Subject: [PATCH 03/56] refactor: replace SELECT * with explicit column lists (#9010) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: replace SELECT * with explicit column lists Co-Authored-By: Claude Opus 4.5 * chore: update sqlx offline query cache * chore: update sqlx offline query cache * chore: update sqlx offline query cache with EE support * chore: update sqlx offline query cache, no deletions * chore: update sqlx offline query cache after rebase * fix: correct column names in explicit script query lists - concurrency_limit → concurrent_limit (matches DB column name) - runnable_settings → runnable_settings_handle (matches DB column name) Co-Authored-By: Claude Opus 4.5 * fix: add missing delete_after_secs column to script queries Also add integration test covering all explicit-column export queries. Co-Authored-By: Claude Opus 4.5 * test: add workspace export integration test covering all explicit-column queries Covers tarball_workspace (folder, script, resource, resource_type, variable, schedule, usr, group_) and the mcp_oauth_client SELECT query from windmill-mcp. Co-Authored-By: Claude Opus 4.5 * test: add tarball export integration test covering all explicit-column queries Single test creates one of each entity type and exercises every runtime-checked explicit-column query in tarball_workspace. Uses archive_type=tar to avoid zip feature-gate in CI. Co-Authored-By: Claude Opus 4.5 * docs: fix stale verification step and CI contradiction in update-sqlx skill - Regenerate current_files.txt after EE cache restoration so step 4 reports accurate diff - Scope "Never use SQLX_OFFLINE=true" to local prepare (CI legitimately uses it) Co-Authored-By: Claude Opus 4.5 * docs: remove Co-Authored-By from commit skill template * refactor: extract SCRIPT_COLUMNS const to single source of truth Replaces 5 duplicated 44-column lists with a shared const in windmill-types. Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 --- .agents/skills/commit/SKILL.md | 4 +- .agents/skills/update-sqlx/SKILL.md | 82 ++++++++ ...6908b75d0d17392c2565e70b4738039b56a1.json} | 4 +- ...5d33d8617fc75c0d9d08e428db488e0be409.json} | 12 +- ...167a2f73a06d661d31c375697a68f7066aa9.json} | 4 +- ...90fdd2f7c4de45117a18360651655d153545.json} | 13 +- ...5b9822bebe88f90ae007d943f2c761858190.json} | 12 +- ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...911a165c9c430b7db389370361ca74d737f4.json} | 13 +- ...3755142647d0257b76e9befd0c4f31c332b1.json} | 4 +- ...3a6cc806b33a844b8e0e600cb8bfae428e84.json} | 4 +- ...e506781295a3bcf036dbbc8600f229e61408.json} | 4 +- ...4dfab99b263e5271045c542c34216e2b47cb.json} | 4 +- ...4d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json} | 12 +- ...6d600e449cd399ac781af8b40c1c444972c3.json} | 12 +- backend/tests/workspace_export.rs | 198 ++++++++++++++++++ backend/windmill-api-configs/src/lib.rs | 4 +- backend/windmill-api-embeddings/src/lib.rs | 2 +- backend/windmill-api-groups/src/groups.rs | 4 +- backend/windmill-api-scripts/src/scripts.rs | 15 +- backend/windmill-api-users/src/users.rs | 4 +- backend/windmill-api/src/workspaces_export.rs | 25 ++- backend/windmill-common/src/scripts.rs | 50 +---- .../windmill-mcp/src/client_registration.rs | 2 +- backend/windmill-queue/src/schedule.rs | 2 +- backend/windmill-store/src/resources.rs | 4 +- backend/windmill-types/src/scripts.rs | 12 ++ 27 files changed, 381 insertions(+), 127 deletions(-) create mode 100644 .agents/skills/update-sqlx/SKILL.md rename backend/.sqlx/{query-ba9ab074f466bc2c581f018e2592f5a453e8a766c35dbf919d29c96966d63c75.json => query-25784f87ccf0bc13b93739d34d416908b75d0d17392c2565e70b4738039b56a1.json} (75%) rename backend/.sqlx/{query-45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7.json => query-27b243e7ff9838a8fc0a4a8e51ac5d33d8617fc75c0d9d08e428db488e0be409.json} (79%) rename backend/.sqlx/{query-d9c8f6ec7bd10e533876526255c15e376ccb4f898b9c0ab8840b2930bda24fdc.json => query-2c9a56fd46767c15dabc7813cd2b167a2f73a06d661d31c375697a68f7066aa9.json} (76%) rename backend/.sqlx/{query-03d63d2e64b012f624d2731b5bcb8849c74a9474777be61edf0ed43ddda07ef3.json => query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json} (81%) rename backend/.sqlx/{query-5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c.json => query-500b68d23314cb0f9edd570e576d5b9822bebe88f90ae007d943f2c761858190.json} (78%) rename backend/.sqlx/{query-b8d392ccfcccafe0c19511b3567bc11779b1052b0948c410468a8aeba1d26d33.json => query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json} (78%) rename backend/.sqlx/{query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json => query-6f95d6927a9be75f6c371c799ebe3755142647d0257b76e9befd0c4f31c332b1.json} (70%) rename backend/.sqlx/{query-e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2.json => query-704d873d4585e00aa0a88cc9949d3a6cc806b33a844b8e0e600cb8bfae428e84.json} (81%) rename backend/.sqlx/{query-60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b.json => query-8f308f841b2d60a6be2bc4ae6aeae506781295a3bcf036dbbc8600f229e61408.json} (83%) rename backend/.sqlx/{query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json => query-c9064664829d304013920e3f710c4dfab99b263e5271045c542c34216e2b47cb.json} (73%) rename backend/.sqlx/{query-7b1239ad6460e8f5fb41bfe12f662a779528784ec8cf3f6dcce5545ab90bf234.json => query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json} (79%) rename backend/.sqlx/{query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json => query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json} (82%) create mode 100644 backend/tests/workspace_export.rs diff --git a/.agents/skills/commit/SKILL.md b/.agents/skills/commit/SKILL.md index d610fa3f65..3f97552466 100644 --- a/.agents/skills/commit/SKILL.md +++ b/.agents/skills/commit/SKILL.md @@ -52,8 +52,6 @@ chore: upgrade sqlx to 0.7 4. Stage ONLY the modified/relevant files: `git add ...` 5. Create the commit with conventional format: ```bash - git commit -m ": - - Co-Authored-By: Claude Opus 4.5 " + git commit -m ": " ``` 6. Run `git status` to verify the commit succeeded diff --git a/.agents/skills/update-sqlx/SKILL.md b/.agents/skills/update-sqlx/SKILL.md new file mode 100644 index 0000000000..ce4eaa5ff7 --- /dev/null +++ b/.agents/skills/update-sqlx/SKILL.md @@ -0,0 +1,82 @@ +--- +name: update-sqlx +description: How to safely update SQLx offline query cache. MUST use when SQL queries change. +--- + +# SQLx Offline Query Cache + +Windmill uses `SQLX_OFFLINE=true` in CI, which requires all `sqlx::query!` / `sqlx::query_as!` macros to have matching cached query data in `backend/.sqlx/`. + +## When to Run + +Run after any change to SQL queries in Rust source files. Without it, CI will fail with: +``` +error: `SQLX_OFFLINE=true` but there is no cached data for this query +``` + +## The Problem + +`cargo sqlx prepare --workspace` **deletes all existing cache files** and regenerates only the ones found in the current compilation. If you don't compile with every feature flag (especially `private` for EE files), you will **silently delete EE query caches**, breaking CI for enterprise tests. + +The standard `./update_sqlx.sh` script tries to compile with all features, but it often fails locally because the EE symlinks can be out of sync with `main`. + +## Safe Procedure + +Always preserve the existing EE caches from `origin/main`. Use this workflow: + +```bash +cd backend + +# 1. Restore the full cache from main (includes EE caches) +git checkout origin/main -- .sqlx/ + +# 2. Run prepare with OSS features (what compiles locally) +# This regenerates OSS caches to match your code changes. +cargo sqlx prepare --workspace -- --workspace --features all_sqlx_features + +# 3. Restore any EE caches that were deleted in step 2. +# These are files present in origin/main but missing after prepare. +git ls-tree origin/main backend/.sqlx/ \ + | awk '{print $4}' | sed 's|backend/\.sqlx/||' | sort > /tmp/main_files.txt + +find backend/.sqlx -name "*.json" -printf '%P\n' | sort > /tmp/current_files.txt + +comm -23 /tmp/main_files.txt /tmp/current_files.txt > /tmp/missing_files.txt + +while read f; do + git show "origin/main:backend/.sqlx/$f" > "backend/.sqlx/$f" +done < /tmp/missing_files.txt + +# 4. Verify nothing was lost from main +find backend/.sqlx -name "*.json" -printf '%P\n' | sort > /tmp/current_files.txt +comm -23 /tmp/main_files.txt /tmp/current_files.txt | wc -l +# Should output: 0 +``` + +## If EE Compiles Locally + +If your EE repo happens to be in sync, you can use the full script (faster): + +```bash +cd backend +./update_sqlx.sh +``` + +But if it fails with EE compilation errors, use the safe procedure above. + +## What NOT to Do + +- **Never** run `cargo sqlx prepare --workspace` with only OSS features and commit the result — it will delete EE caches. +- **Never** set `SQLX_OFFLINE=true` for local `cargo sqlx prepare` — use a live database per CLAUDE.md. (CI runs with `SQLX_OFFLINE=true`, which is why the cache must be complete.) +- **Never** skip the verification step (step 4 above). + +## Verification + +After committing, the diff against `origin/main` should show: +- A few **new** cache files (for your changed queries) +- A few **deleted** cache files (for old queries that no longer exist) +- **Zero** net deletions from the EE cache set + +```bash +git diff origin/main --stat backend/.sqlx/ +``` diff --git a/backend/.sqlx/query-ba9ab074f466bc2c581f018e2592f5a453e8a766c35dbf919d29c96966d63c75.json b/backend/.sqlx/query-25784f87ccf0bc13b93739d34d416908b75d0d17392c2565e70b4738039b56a1.json similarity index 75% rename from backend/.sqlx/query-ba9ab074f466bc2c581f018e2592f5a453e8a766c35dbf919d29c96966d63c75.json rename to backend/.sqlx/query-25784f87ccf0bc13b93739d34d416908b75d0d17392c2565e70b4738039b56a1.json index c7114f6591..8cae07c761 100644 --- a/backend/.sqlx/query-ba9ab074f466bc2c581f018e2592f5a453e8a766c35dbf919d29c96966d63c75.json +++ b/backend/.sqlx/query-25784f87ccf0bc13b93739d34d416908b75d0d17392c2565e70b4738039b56a1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * FROM group_ WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", + "query": "SELECT workspace_id, name, summary, extra_perms FROM group_ WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -38,5 +38,5 @@ false ] }, - "hash": "ba9ab074f466bc2c581f018e2592f5a453e8a766c35dbf919d29c96966d63c75" + "hash": "25784f87ccf0bc13b93739d34d416908b75d0d17392c2565e70b4738039b56a1" } diff --git a/backend/.sqlx/query-45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7.json b/backend/.sqlx/query-27b243e7ff9838a8fc0a4a8e51ac5d33d8617fc75c0d9d08e428db488e0be409.json similarity index 79% rename from backend/.sqlx/query-45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7.json rename to backend/.sqlx/query-27b243e7ff9838a8fc0a4a8e51ac5d33d8617fc75c0d9d08e428db488e0be409.json index 12b8201ebb..ba1ffe0ac6 100644 --- a/backend/.sqlx/query-45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7.json +++ b/backend/.sqlx/query-27b243e7ff9838a8fc0a4a8e51ac5d33d8617fc75c0d9d08e428db488e0be409.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * FROM resource WHERE workspace_id = $1 AND resource_type != 'state' AND resource_type != 'cache'", + "query": "SELECT workspace_id, path, value, description, resource_type, extra_perms, created_by, edited_at, labels FROM resource WHERE workspace_id = $1 AND resource_type != 'state' AND resource_type != 'cache'", "describe": { "columns": [ { @@ -35,13 +35,13 @@ }, { "ordinal": 6, - "name": "edited_at", - "type_info": "Timestamptz" + "name": "created_by", + "type_info": "Varchar" }, { "ordinal": 7, - "name": "created_by", - "type_info": "Varchar" + "name": "edited_at", + "type_info": "Timestamptz" }, { "ordinal": 8, @@ -66,5 +66,5 @@ true ] }, - "hash": "45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7" + "hash": "27b243e7ff9838a8fc0a4a8e51ac5d33d8617fc75c0d9d08e428db488e0be409" } diff --git a/backend/.sqlx/query-d9c8f6ec7bd10e533876526255c15e376ccb4f898b9c0ab8840b2930bda24fdc.json b/backend/.sqlx/query-2c9a56fd46767c15dabc7813cd2b167a2f73a06d661d31c375697a68f7066aa9.json similarity index 76% rename from backend/.sqlx/query-d9c8f6ec7bd10e533876526255c15e376ccb4f898b9c0ab8840b2930bda24fdc.json rename to backend/.sqlx/query-2c9a56fd46767c15dabc7813cd2b167a2f73a06d661d31c375697a68f7066aa9.json index 463a3f32f0..7ca04a8311 100644 --- a/backend/.sqlx/query-d9c8f6ec7bd10e533876526255c15e376ccb4f898b9c0ab8840b2930bda24fdc.json +++ b/backend/.sqlx/query-2c9a56fd46767c15dabc7813cd2b167a2f73a06d661d31c375697a68f7066aa9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * FROM group_ WHERE name = $1 AND workspace_id = $2", + "query": "SELECT workspace_id, name, summary, extra_perms FROM group_ WHERE name = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -37,5 +37,5 @@ false ] }, - "hash": "d9c8f6ec7bd10e533876526255c15e376ccb4f898b9c0ab8840b2930bda24fdc" + "hash": "2c9a56fd46767c15dabc7813cd2b167a2f73a06d661d31c375697a68f7066aa9" } diff --git a/backend/.sqlx/query-03d63d2e64b012f624d2731b5bcb8849c74a9474777be61edf0ed43ddda07ef3.json b/backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json similarity index 81% rename from backend/.sqlx/query-03d63d2e64b012f624d2731b5bcb8849c74a9474777be61edf0ed43ddda07ef3.json rename to backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json index 31ec682852..e4db87ec7d 100644 --- a/backend/.sqlx/query-03d63d2e64b012f624d2731b5bcb8849c74a9474777be61edf0ed43ddda07ef3.json +++ b/backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1", "describe": { "columns": [ { @@ -25,13 +25,13 @@ }, { "ordinal": 4, - "name": "edited_at", - "type_info": "Timestamptz" + "name": "created_by", + "type_info": "Varchar" }, { "ordinal": 5, - "name": "created_by", - "type_info": "Varchar" + "name": "edited_at", + "type_info": "Timestamptz" }, { "ordinal": 6, @@ -46,7 +46,6 @@ ], "parameters": { "Left": [ - "Text", "Text" ] }, @@ -61,5 +60,5 @@ false ] }, - "hash": "03d63d2e64b012f624d2731b5bcb8849c74a9474777be61edf0ed43ddda07ef3" + "hash": "45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545" } diff --git a/backend/.sqlx/query-5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c.json b/backend/.sqlx/query-500b68d23314cb0f9edd570e576d5b9822bebe88f90ae007d943f2c761858190.json similarity index 78% rename from backend/.sqlx/query-5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c.json rename to backend/.sqlx/query-500b68d23314cb0f9edd570e576d5b9822bebe88f90ae007d943f2c761858190.json index 79625b6baf..bb8731d445 100644 --- a/backend/.sqlx/query-5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c.json +++ b/backend/.sqlx/query-500b68d23314cb0f9edd570e576d5b9822bebe88f90ae007d943f2c761858190.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT *\n FROM usr\n WHERE workspace_id = $1\n ", + "query": "SELECT workspace_id, username, email, is_admin, created_at, operator, disabled, role, added_via FROM usr\n WHERE workspace_id = $1", "describe": { "columns": [ { @@ -47,11 +47,6 @@ "ordinal": 8, "name": "added_via", "type_info": "Jsonb" - }, - { - "ordinal": 9, - "name": "is_service_account", - "type_info": "Bool" } ], "parameters": { @@ -68,9 +63,8 @@ false, false, true, - true, - false + true ] }, - "hash": "5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c" + "hash": "500b68d23314cb0f9edd570e576d5b9822bebe88f90ae007d943f2c761858190" } diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-b8d392ccfcccafe0c19511b3567bc11779b1052b0948c410468a8aeba1d26d33.json b/backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json similarity index 78% rename from backend/.sqlx/query-b8d392ccfcccafe0c19511b3567bc11779b1052b0948c410468a8aeba1d26d33.json rename to backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json index 4871744e03..d2084d76d8 100644 --- a/backend/.sqlx/query-b8d392ccfcccafe0c19511b3567bc11779b1052b0948c410468a8aeba1d26d33.json +++ b/backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER BY name", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", "describe": { "columns": [ { @@ -25,13 +25,13 @@ }, { "ordinal": 4, - "name": "edited_at", - "type_info": "Timestamptz" + "name": "created_by", + "type_info": "Varchar" }, { "ordinal": 5, - "name": "created_by", - "type_info": "Varchar" + "name": "edited_at", + "type_info": "Timestamptz" }, { "ordinal": 6, @@ -46,6 +46,7 @@ ], "parameters": { "Left": [ + "Text", "Text" ] }, @@ -60,5 +61,5 @@ false ] }, - "hash": "b8d392ccfcccafe0c19511b3567bc11779b1052b0948c410468a8aeba1d26d33" + "hash": "623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4" } diff --git a/backend/.sqlx/query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json b/backend/.sqlx/query-6f95d6927a9be75f6c371c799ebe3755142647d0257b76e9befd0c4f31c332b1.json similarity index 70% rename from backend/.sqlx/query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json rename to backend/.sqlx/query-6f95d6927a9be75f6c371c799ebe3755142647d0257b76e9befd0c4f31c332b1.json index b64a1eb468..c97cc9ff8d 100644 --- a/backend/.sqlx/query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json +++ b/backend/.sqlx/query-6f95d6927a9be75f6c371c799ebe3755142647d0257b76e9befd0c4f31c332b1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * FROM config WHERE name LIKE 'worker__%'", + "query": "SELECT name, config FROM config WHERE name LIKE 'worker__%'", "describe": { "columns": [ { @@ -22,5 +22,5 @@ true ] }, - "hash": "ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7" + "hash": "6f95d6927a9be75f6c371c799ebe3755142647d0257b76e9befd0c4f31c332b1" } diff --git a/backend/.sqlx/query-e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2.json b/backend/.sqlx/query-704d873d4585e00aa0a88cc9949d3a6cc806b33a844b8e0e600cb8bfae428e84.json similarity index 81% rename from backend/.sqlx/query-e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2.json rename to backend/.sqlx/query-704d873d4585e00aa0a88cc9949d3a6cc806b33a844b8e0e600cb8bfae428e84.json index 7be961c050..9f80167752 100644 --- a/backend/.sqlx/query-e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2.json +++ b/backend/.sqlx/query-704d873d4585e00aa0a88cc9949d3a6cc806b33a844b8e0e600cb8bfae428e84.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * FROM usr\n WHERE workspace_id = $1", + "query": "\n SELECT workspace_id, username, email, is_admin, created_at, operator, disabled, role, added_via, is_service_account\n FROM usr\n WHERE workspace_id = $1\n ", "describe": { "columns": [ { @@ -72,5 +72,5 @@ false ] }, - "hash": "e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2" + "hash": "704d873d4585e00aa0a88cc9949d3a6cc806b33a844b8e0e600cb8bfae428e84" } diff --git a/backend/.sqlx/query-60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b.json b/backend/.sqlx/query-8f308f841b2d60a6be2bc4ae6aeae506781295a3bcf036dbbc8600f229e61408.json similarity index 83% rename from backend/.sqlx/query-60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b.json rename to backend/.sqlx/query-8f308f841b2d60a6be2bc4ae6aeae506781295a3bcf036dbbc8600f229e61408.json index ed09f2833f..920ce385f3 100644 --- a/backend/.sqlx/query-60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b.json +++ b/backend/.sqlx/query-8f308f841b2d60a6be2bc4ae6aeae506781295a3bcf036dbbc8600f229e61408.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * FROM usr WHERE username = $1 AND workspace_id = $2", + "query": "SELECT workspace_id, username, email, is_admin, created_at, operator, disabled, role, added_via, is_service_account FROM usr WHERE username = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -73,5 +73,5 @@ false ] }, - "hash": "60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b" + "hash": "8f308f841b2d60a6be2bc4ae6aeae506781295a3bcf036dbbc8600f229e61408" } diff --git a/backend/.sqlx/query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json b/backend/.sqlx/query-c9064664829d304013920e3f710c4dfab99b263e5271045c542c34216e2b47cb.json similarity index 73% rename from backend/.sqlx/query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json rename to backend/.sqlx/query-c9064664829d304013920e3f710c4dfab99b263e5271045c542c34216e2b47cb.json index b235a60c1a..1eb8cf3595 100644 --- a/backend/.sqlx/query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json +++ b/backend/.sqlx/query-c9064664829d304013920e3f710c4dfab99b263e5271045c542c34216e2b47cb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * FROM config WHERE name = $1", + "query": "SELECT name, config FROM config WHERE name = $1", "describe": { "columns": [ { @@ -24,5 +24,5 @@ true ] }, - "hash": "d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9" + "hash": "c9064664829d304013920e3f710c4dfab99b263e5271045c542c34216e2b47cb" } diff --git a/backend/.sqlx/query-7b1239ad6460e8f5fb41bfe12f662a779528784ec8cf3f6dcce5545ab90bf234.json b/backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json similarity index 79% rename from backend/.sqlx/query-7b1239ad6460e8f5fb41bfe12f662a779528784ec8cf3f6dcce5545ab90bf234.json rename to backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json index 29f4baacc2..ffd1670c04 100644 --- a/backend/.sqlx/query-7b1239ad6460e8f5fb41bfe12f662a779528784ec8cf3f6dcce5545ab90bf234.json +++ b/backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * FROM resource_type WHERE workspace_id = $1", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER BY name", "describe": { "columns": [ { @@ -25,13 +25,13 @@ }, { "ordinal": 4, - "name": "edited_at", - "type_info": "Timestamptz" + "name": "created_by", + "type_info": "Varchar" }, { "ordinal": 5, - "name": "created_by", - "type_info": "Varchar" + "name": "edited_at", + "type_info": "Timestamptz" }, { "ordinal": 6, @@ -60,5 +60,5 @@ false ] }, - "hash": "7b1239ad6460e8f5fb41bfe12f662a779528784ec8cf3f6dcce5545ab90bf234" + "hash": "d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6" } diff --git a/backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json b/backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json similarity index 82% rename from backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json rename to backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json index ce532af385..c44d3d711d 100644 --- a/backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json +++ b/backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * from resource_type ORDER BY name", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type ORDER BY name", "describe": { "columns": [ { @@ -25,13 +25,13 @@ }, { "ordinal": 4, - "name": "edited_at", - "type_info": "Timestamptz" + "name": "created_by", + "type_info": "Varchar" }, { "ordinal": 5, - "name": "created_by", - "type_info": "Varchar" + "name": "edited_at", + "type_info": "Timestamptz" }, { "ordinal": 6, @@ -58,5 +58,5 @@ false ] }, - "hash": "eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12" + "hash": "e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3" } diff --git a/backend/tests/workspace_export.rs b/backend/tests/workspace_export.rs new file mode 100644 index 0000000000..823ff22a43 --- /dev/null +++ b/backend/tests/workspace_export.rs @@ -0,0 +1,198 @@ +use sqlx::postgres::Postgres; +use sqlx::Pool; +use windmill_test_utils::{initialize_tracing, ApiServer}; + +/// Integration test: exercises every explicit-column query in `tarball_workspace`. +/// +/// Creates one entity of each type (folder, script, resource, resource_type, +/// variable, schedule, group) in the test workspace, then calls the tarball +/// export endpoint with all include_* flags enabled. Success means every +/// `SELECT col1, col2, ...` list matches the database schema. +/// +/// Tables exercised (one explicit-column query each): +/// folder, script, resource, resource_type, variable, schedule, usr, group_ +#[sqlx::test(fixtures("base"))] +async fn test_tarball_export_all_tables(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base_url = format!("http://localhost:{port}"); + + let client = windmill_api_client::create_client(&base_url, "SECRET_TOKEN".to_string()); + let http = client.client(); + + // ---- folder ---- + sqlx::query( + r#"INSERT INTO folder + (workspace_id, name, display_name, owners, extra_perms, summary) + VALUES ($1, $2, $3, $4, '{}'::jsonb, $5)"#, + ) + .bind("test-workspace") + .bind("test_folder") + .bind("Test Folder") + .bind(vec!["u/test-user"]) + .bind("a test folder") + .execute(&db) + .await?; + + // ---- script (exercises the 30-column Script query) ---- + client + .create_script( + "test-workspace", + &windmill_api_client::types::NewScript { + content: "export function main() { return 42; }".to_string(), + language: windmill_api_client::types::ScriptLang::Bun, + path: "f/test_folder/test_script".to_string(), + summary: "test script".to_string(), + description: "script for export test".to_string(), + kind: Some("script".to_string()), + tag: Some("test".to_string()), + lock: None, + parent_hash: None, + schema: Default::default(), + is_template: None, + draft_only: None, + dedicated_worker: None, + ws_error_handler_muted: None, + priority: None, + cache_ttl: None, + concurrent_limit: None, + concurrency_time_window_s: None, + timeout: None, + delete_after_secs: None, + restart_unless_cancelled: None, + visible_to_runner_only: None, + auto_kind: None, + on_behalf_of_email: None, + has_preprocessor: None, + codebase: None, + envs: vec![], + deployment_message: None, + assets: vec![], + modules: None, + concurrency_key: None, + }, + ) + .await?; + + // ---- resource ---- + sqlx::query( + r#"INSERT INTO resource + (workspace_id, path, value, description, resource_type, created_by) + VALUES ($1, $2, $3, $4, $5, $6)"#, + ) + .bind("test-workspace") + .bind("f/test_folder/test_res") + .bind(serde_json::json!({"url": "http://example.com"})) + .bind("test resource") + .bind("http") + .bind("test-user") + .execute(&db) + .await?; + + // ---- resource_type ---- + sqlx::query( + r#"INSERT INTO resource_type + (workspace_id, name, schema, description, created_by) + VALUES ($1, $2, $3, $4, $5)"#, + ) + .bind("test-workspace") + .bind("http") + .bind(serde_json::json!({"type": "object"})) + .bind("HTTP resource type") + .bind("system") + .execute(&db) + .await?; + + // ---- variable ---- + sqlx::query( + r#"INSERT INTO variable + (workspace_id, path, value, is_secret, description, account) + VALUES ($1, $2, $3, $4, $5, $6)"#, + ) + .bind("test-workspace") + .bind("f/test_folder/test_var") + .bind("test_value") + .bind(false) + .bind("test variable") + .bind(None::) + .execute(&db) + .await?; + + // ---- schedule ---- + client + .create_schedule( + "test-workspace", + &windmill_api_client::types::NewSchedule { + schedule: "0 0 0 * * *".to_string(), + script_path: "f/test_folder/test_script".to_string(), + path: "f/test_folder/test_schedule".to_string(), + is_flow: false, + timezone: "UTC".to_string(), + args: Default::default(), + enabled: Some(false), + description: Some("test schedule".to_string()), + summary: Some("test schedule".to_string()), + tag: None, + cron_version: Some("v2".to_string()), + on_failure: None, + on_failure_times: None, + on_failure_exact: None, + on_failure_extra_args: None, + on_recovery: None, + on_recovery_times: None, + on_recovery_extra_args: None, + on_success: None, + on_success_extra_args: None, + ws_error_handler_muted: None, + retry: None, + no_flow_overlap: None, + }, + ) + .await?; + + // ---- group_ (base fixture already has "all", create one more for include_groups) ---- + sqlx::query( + "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, '{}'::jsonb)", + ) + .bind("test-workspace") + .bind("testgroup") + .bind("test group") + .execute(&db) + .await?; + + // ---- tarball export: hits ALL explicit-column queries at once ---- + let params = [ + "archive_type=tar", + "include_schedules=true", + "include_users=true", + "include_groups=true", + "include_settings=true", + "include_workspace_dependencies=true", + "settings_version=v1", + ]; + + let resp = http + .get(format!( + "{}/api/w/test-workspace/workspaces/tarball?{}", + base_url, + params.join("&") + )) + .bearer_auth("SECRET_TOKEN") + .send() + .await?; + + assert_eq!( + resp.status(), + 200, + "tarball export failed: {}", + resp.text().await.unwrap_or_default() + ); + + // Verify we got actual bytes back + let body = resp.bytes().await?; + assert!(!body.is_empty(), "tarball export returned empty body"); + + Ok(()) +} + diff --git a/backend/windmill-api-configs/src/lib.rs b/backend/windmill-api-configs/src/lib.rs index 0901e73e67..e18afc35c8 100644 --- a/backend/windmill-api-configs/src/lib.rs +++ b/backend/windmill-api-configs/src/lib.rs @@ -64,7 +64,7 @@ async fn list_worker_groups( Extension(db): Extension, ) -> error::JsonResult> { let mut configs_raw = - sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'") + sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name LIKE 'worker__%'") .fetch_all(&db) .await?; // Remove the 'worker__' prefix from all config names @@ -119,7 +119,7 @@ async fn get_config( ) -> error::JsonResult> { require_devops_role(&db, &authed.email).await?; - let config = sqlx::query_as!(Config, "SELECT * FROM config WHERE name = $1", name) + let config = sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name = $1", name) .fetch_optional(&db) .await? .map(|c| c.config); diff --git a/backend/windmill-api-embeddings/src/lib.rs b/backend/windmill-api-embeddings/src/lib.rs index ae8eca67cf..f0c87633d2 100644 --- a/backend/windmill-api-embeddings/src/lib.rs +++ b/backend/windmill-api-embeddings/src/lib.rs @@ -365,7 +365,7 @@ impl EmbeddingsDb { let hub_resource_types = response.json::>().await?; let resource_types: Vec = - sqlx::query_as!(ResourceType, "SELECT * from resource_type ORDER BY name",) + sqlx::query_as!(ResourceType, "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type ORDER BY name",) .fetch_all(pg_db) .await?; diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index af67f884c9..263daad624 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -109,7 +109,7 @@ async fn list_groups( let rows = sqlx::query_as!( Group, - "SELECT * FROM group_ WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", + "SELECT workspace_id, name, summary, extra_perms FROM group_ WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", w_id, per_page as i64, offset as i64 @@ -574,7 +574,7 @@ pub async fn get_group_opt<'c>( ) -> Result> { let group_opt = sqlx::query_as!( Group, - "SELECT * FROM group_ WHERE name = $1 AND workspace_id = $2", + "SELECT workspace_id, name, summary, extra_perms FROM group_ WHERE name = $1 AND workspace_id = $2", name, w_id ) diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 426b92ac17..de0f38cd34 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -979,7 +979,10 @@ async fn create_script_internal<'c>( .await?; } let clashing_script = sqlx::query_as::<_, Script>( - "SELECT * FROM script WHERE path = $1 AND archived = false AND workspace_id = $2", + &format!( + "SELECT {} FROM script WHERE path = $1 AND archived = false AND workspace_id = $2", + windmill_common::scripts::SCRIPT_COLUMNS, + ), ) .bind(&ns.path) .bind(&w_id) @@ -1762,7 +1765,10 @@ async fn get_script_by_path( .await? } else { sqlx::query_as::<_, ScriptWithStarred>( - "SELECT *, NULL as starred FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + &format!( + "SELECT {}, NULL as starred FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + windmill_common::scripts::SCRIPT_COLUMNS, + ), ) .bind(path) .bind(w_id) @@ -2277,7 +2283,10 @@ async fn get_script_by_hash_internal<'c>( .await? } else { sqlx::query_as::<_, ScriptWithStarred>( - "SELECT *, NULL as starred FROM script WHERE hash = $1 AND workspace_id = $2", + &format!( + "SELECT {}, NULL as starred FROM script WHERE hash = $1 AND workspace_id = $2", + windmill_common::scripts::SCRIPT_COLUMNS, + ), ) .bind(hash) .bind(workspace_id) diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index f664ab09b8..382a69201b 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -385,7 +385,7 @@ async fn list_users( let rows = sqlx::query_as!( User, " - SELECT * + SELECT workspace_id, username, email, is_admin, created_at, operator, disabled, role, added_via, is_service_account FROM usr WHERE workspace_id = $1 ", @@ -1174,7 +1174,7 @@ async fn get_workspace_user( let user = sqlx::query_as!( User, - "SELECT * FROM usr WHERE username = $1 AND workspace_id = $2", + "SELECT workspace_id, username, email, is_admin, created_at, operator, disabled, role, added_via, is_service_account FROM usr WHERE username = $1 AND workspace_id = $2", &username_to_update, &w_id ) diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 91b9ab972e..e6b3b9ee0f 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -490,7 +490,7 @@ pub(crate) async fn tarball_workspace( Some(t) => Err(Error::BadRequest(format!("Invalid Archive Type {t}"))), }?; { - let folders = sqlx::query_as::<_, Folder>("SELECT * FROM folder WHERE workspace_id = $1") + let folders = sqlx::query_as::<_, Folder>("SELECT name, workspace_id, display_name, owners, extra_perms, summary, edited_at, created_by, default_permissioned_as FROM folder WHERE workspace_id = $1") .bind(&w_id) .fetch_all(&mut *tx) .await?; @@ -507,10 +507,13 @@ pub(crate) async fn tarball_workspace( { let scripts = sqlx::query_as::<_, Script>( - "SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false - AND (draft_only IS NULL OR draft_only = false) - AND created_at = (select max(created_at) from script where path = o.path AND \ - workspace_id = $1)", + &format!( + "SELECT {} FROM script as o WHERE workspace_id = $1 AND archived = false + AND (draft_only IS NULL OR draft_only = false) + AND created_at = (select max(created_at) from script where path = o.path AND \ + workspace_id = $1)", + windmill_common::scripts::SCRIPT_COLUMNS, + ), ) .bind(&w_id) .fetch_all(&mut *tx) @@ -595,7 +598,7 @@ pub(crate) async fn tarball_workspace( if !skip_resources.unwrap_or(false) { let resources = sqlx::query_as!( Resource, - "SELECT * FROM resource WHERE workspace_id = $1 AND resource_type != 'state' AND resource_type != 'cache'", + "SELECT workspace_id, path, value, description, resource_type, extra_perms, created_by, edited_at, labels FROM resource WHERE workspace_id = $1 AND resource_type != 'state' AND resource_type != 'cache'", &w_id ) .fetch_all(&mut *tx) @@ -612,7 +615,7 @@ pub(crate) async fn tarball_workspace( if !skip_resource_types.unwrap_or(false) { let resource_types = sqlx::query_as!( ResourceType, - "SELECT * FROM resource_type WHERE workspace_id = $1", + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1", &w_id ) .fetch_all(&mut *tx) @@ -651,9 +654,9 @@ pub(crate) async fn tarball_workspace( if !skip_variables.unwrap_or(false) { let variables = sqlx::query_as::<_, ExportableListableVariable>(if !skip_secrets.unwrap_or(false) { - "SELECT * FROM variable WHERE workspace_id = $1 AND expires_at IS NULL" + "SELECT workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at, labels FROM variable WHERE workspace_id = $1 AND expires_at IS NULL" } else { - "SELECT * FROM variable WHERE workspace_id = $1 AND is_secret = false AND expires_at IS NULL" + "SELECT workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at, labels FROM variable WHERE workspace_id = $1 AND is_secret = false AND expires_at IS NULL" }) .bind(&w_id) .fetch_all(&mut *tx) @@ -727,7 +730,7 @@ pub(crate) async fn tarball_workspace( if include_schedules.unwrap_or(false) { let schedules = sqlx::query_as::<_, Schedule>( - "SELECT * FROM schedule + "SELECT workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, args, extra_perms, email, permissioned_as, error, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, on_success, on_success_extra_args, ws_error_handler_muted, retry, no_flow_overlap, summary, description, tag, paused_until, cron_version, dynamic_skip, labels FROM schedule WHERE workspace_id = $1", ) .bind(&w_id) @@ -998,7 +1001,7 @@ pub(crate) async fn tarball_workspace( if include_users.unwrap_or(false) { let users = sqlx::query!( - "SELECT * FROM usr + "SELECT workspace_id, username, email, is_admin, created_at, operator, disabled, role, added_via FROM usr WHERE workspace_id = $1", &w_id ) diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 192085d902..dd4c0bb42e 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -312,52 +312,10 @@ pub async fn fetch_script_for_update<'a>( e: impl sqlx::Executor<'a, Database = sqlx::Postgres>, ) -> crate::error::Result>> { sqlx::query_as::<_, Script>( - "SELECT - workspace_id, - hash, - path, - parent_hashes, - summary, - description, - content, - created_by, - created_at, - archived, - schema, - deleted, - is_template, - extra_perms, - lock, - lock_error_logs, - language, - kind, - tag, - draft_only, - envs, - concurrency_key, - concurrent_limit, - concurrency_time_window_s, - debounce_key, - debounce_delay_s, - dedicated_worker, - runnable_settings_handle, - ws_error_handler_muted, - priority, - cache_ttl, - cache_ignore_s3_path, - timeout, - delete_after_use, - delete_after_secs, - restart_unless_cancelled, - visible_to_runner_only, - auto_kind, - codebase, - has_preprocessor, - on_behalf_of_email, - assets, - modules, - labels - FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1 FOR UPDATE", + &format!( + "SELECT {} FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1 FOR UPDATE", + SCRIPT_COLUMNS, + ), ) .bind(path) .bind(w_id) diff --git a/backend/windmill-mcp/src/client_registration.rs b/backend/windmill-mcp/src/client_registration.rs index 56f992540a..eca4ed650f 100644 --- a/backend/windmill-mcp/src/client_registration.rs +++ b/backend/windmill-mcp/src/client_registration.rs @@ -122,7 +122,7 @@ pub async fn get_or_refresh_mcp_client( let redirect_uri = format!("{}/api/mcp/oauth/callback", base_url); let cached_client: Option = - sqlx::query_as("SELECT * FROM mcp_oauth_client WHERE mcp_server_url = $1") + sqlx::query_as("SELECT mcp_server_url, client_id, client_secret, client_secret_expires_at, token_endpoint FROM mcp_oauth_client WHERE mcp_server_url = $1") .bind(mcp_server_url) .fetch_optional(db) .await diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 6b0a9904b5..3f6c587be2 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -566,7 +566,7 @@ pub async fn get_schedule_opt<'c>( path: &str, ) -> Result> { let schedule_opt = sqlx::query_as::<_, Schedule>( - "SELECT * FROM schedule WHERE path = $1 AND workspace_id = $2", + "SELECT workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, args, extra_perms, email, permissioned_as, error, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, on_success, on_success_extra_args, ws_error_handler_muted, retry, no_flow_overlap, summary, description, tag, paused_until, cron_version, dynamic_skip, labels FROM schedule WHERE path = $1 AND workspace_id = $2", ) .bind(path) .bind(w_id) diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index a331542acc..ed471bfaa7 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -1510,7 +1510,7 @@ async fn list_resource_types( ) -> JsonResult> { let rows = sqlx::query_as!( ResourceType, - "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ BY name", &w_id ) @@ -1544,7 +1544,7 @@ async fn get_resource_type( let resource_type_o = sqlx::query_as!( ResourceType, - "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", &name, &w_id ) diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index 7bf19e6147..323ca80b13 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -318,6 +318,18 @@ pub fn id_to_codebase_info(id: &str) -> CodebaseInfo { CodebaseInfo { is_tar, is_esm } } +/// Column list for `SELECT ... FROM script` when targeting `Script`. +/// Shared across all query sites so a schema change only needs one edit. +pub const SCRIPT_COLUMNS: &str = "workspace_id, hash, path, parent_hashes, summary, \ + description, content, created_by, created_at, archived, schema, deleted, \ + is_template, extra_perms, lock, lock_error_logs, language, kind, tag, \ + draft_only, envs, dedicated_worker, ws_error_handler_muted, priority, \ + cache_ttl, cache_ignore_s3_path, timeout, delete_after_use, delete_after_secs, \ + restart_unless_cancelled, visible_to_runner_only, auto_kind, codebase, \ + has_preprocessor, on_behalf_of_email, assets, modules, labels, \ + concurrency_key, concurrent_limit, concurrency_time_window_s, \ + debounce_key, debounce_delay_s, runnable_settings_handle"; + #[derive(Serialize, sqlx::FromRow, Debug)] pub struct Script { pub workspace_id: String, From 26a41df3e33317d8ed5544a08fcfb711a07ed8f8 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 4 May 2026 08:44:15 +0000 Subject: [PATCH 04/56] chore: use concat! for SCRIPT_COLUMNS to avoid \ continuation whitespace --- backend/windmill-types/src/scripts.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index 323ca80b13..e9e9337e6d 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -320,15 +320,16 @@ pub fn id_to_codebase_info(id: &str) -> CodebaseInfo { /// Column list for `SELECT ... FROM script` when targeting `Script`. /// Shared across all query sites so a schema change only needs one edit. -pub const SCRIPT_COLUMNS: &str = "workspace_id, hash, path, parent_hashes, summary, \ - description, content, created_by, created_at, archived, schema, deleted, \ - is_template, extra_perms, lock, lock_error_logs, language, kind, tag, \ - draft_only, envs, dedicated_worker, ws_error_handler_muted, priority, \ - cache_ttl, cache_ignore_s3_path, timeout, delete_after_use, delete_after_secs, \ - restart_unless_cancelled, visible_to_runner_only, auto_kind, codebase, \ - has_preprocessor, on_behalf_of_email, assets, modules, labels, \ - concurrency_key, concurrent_limit, concurrency_time_window_s, \ - debounce_key, debounce_delay_s, runnable_settings_handle"; +pub const SCRIPT_COLUMNS: &str = concat!( + "workspace_id, hash, path, parent_hashes, summary, description, content, ", + "created_by, created_at, archived, schema, deleted, is_template, extra_perms, ", + "lock, lock_error_logs, language, kind, tag, draft_only, envs, ", + "dedicated_worker, ws_error_handler_muted, priority, cache_ttl, cache_ignore_s3_path, ", + "timeout, delete_after_use, delete_after_secs, restart_unless_cancelled, ", + "visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of_email, ", + "assets, modules, labels, concurrency_key, concurrent_limit, ", + "concurrency_time_window_s, debounce_key, debounce_delay_s, runnable_settings_handle", +); #[derive(Serialize, sqlx::FromRow, Debug)] pub struct Script { From 44fad139fe2076f5faa62de31aaeeb217466b25e Mon Sep 17 00:00:00 2001 From: Ayush Baluni <73417844+aayushbaluni@users.noreply.github.com> Date: Mon, 4 May 2026 14:20:14 +0530 Subject: [PATCH 05/56] fix: bind MySQL table listing to configured database name (#9007) * fix: bind MySQL table listing to configured database name * refactor: drop DATABASE() sentinel and cover single-table fallback Co-Authored-By: Claude Opus 4.7 (1M context) * refactor: unify MySQL schema-resolution branches via explicit_db binding Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Ruben Fiszel Co-authored-by: Claude Opus 4.7 (1M context) --- backend/windmill-common/src/query_builders.rs | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index e51d24fa44..f077687d6f 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -2430,18 +2430,30 @@ WHERE table_schema = current_schema()", Ok(q) } DbType::Mysql => { - let db_name = database_name.unwrap_or(""); + let explicit_db = database_name.filter(|s| !s.is_empty()); let table_filter = if let Some(t) = table { let parts: Vec<&str> = t.split('.').collect(); let tname = parts[parts.len() - 1]; - let schema = if parts.len() > 1 { parts[0] } else { db_name }; + let schema_sql = if parts.len() > 1 { + format!("'{}'", escape_sql_literal(parts[0])) + } else { + explicit_db + .map(|dn| format!("'{}'", escape_sql_literal(dn))) + .unwrap_or_else(|| "DATABASE()".to_string()) + }; format!( - "\nWHERE\n TABLE_NAME = '{}' AND TABLE_SCHEMA = '{}'", + "\nWHERE\n TABLE_NAME = '{}' AND TABLE_SCHEMA = {}", escape_sql_literal(tname), - escape_sql_literal(schema) + schema_sql ) } else { - "\nWHERE\n TABLE_SCHEMA NOT IN ('mysql', 'performance_schema', 'information_schema', 'sys', '_vt')".to_string() + let schema_predicate = explicit_db + .map(|dn| format!("TABLE_SCHEMA = '{}'", escape_sql_literal(dn))) + .unwrap_or_else(|| "TABLE_SCHEMA = DATABASE()".to_string()); + format!( + "\nWHERE\n {}\n AND TABLE_SCHEMA NOT IN ('mysql', 'performance_schema', 'information_schema', 'sys', '_vt')", + schema_predicate + ) }; let extra_col = if table.is_none() { ",\n TABLE_NAME as table_name" @@ -4250,10 +4262,28 @@ mod tests { } #[test] - fn test_expand_load_table_metadata_mysql_all_tables() { + fn test_expand_load_table_metadata_mysql_single_table_uses_session_db() { + let marker = r#"-- WM_INTERNAL_DB_LOAD_TABLE_METADATA {"table":"users"}"#; + let sql = expand_code(marker, &ScriptLang::Mysql); + assert!(sql.contains("TABLE_NAME = 'users'")); + assert!(sql.contains("TABLE_SCHEMA = DATABASE()")); + } + + #[test] + fn test_expand_load_table_metadata_mysql_all_tables_uses_session_db() { let marker = r#"-- WM_INTERNAL_DB_LOAD_TABLE_METADATA {}"#; let sql = expand_code(marker, &ScriptLang::Mysql); assert!(sql.contains("TABLE_NAME as table_name")); + assert!(sql.contains("TABLE_SCHEMA = DATABASE()")); + assert!(sql.contains("TABLE_SCHEMA NOT IN")); + } + + #[test] + fn test_expand_load_table_metadata_mysql_all_tables_with_database_name() { + let marker = r#"-- WM_INTERNAL_DB_LOAD_TABLE_METADATA {"databaseName":"mydb"}"#; + let sql = expand_code(marker, &ScriptLang::Mysql); + assert!(sql.contains("TABLE_NAME as table_name")); + assert!(sql.contains("TABLE_SCHEMA = 'mydb'")); assert!(sql.contains("TABLE_SCHEMA NOT IN")); } From 8529a2cf1a7776d1be6d4655954e60bf99fbb699 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Mon, 4 May 2026 15:19:03 +0200 Subject: [PATCH 06/56] chore(frontend): expose DarkModeObserver, TextInput, common/Badge from windmill-components (#9018) Adds three subpath entries to the windmill-components package's `exports` and `typesVersions` so external consumers (e.g. windmillhub) can import these components without resorting to private `node_modules` aliases. Co-authored-by: Claude Opus 4.7 (1M context) --- frontend/package.json | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/frontend/package.json b/frontend/package.json index b578298993..852edacb50 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -426,6 +426,21 @@ "./tailwindUtils": { "types": "./package/components/apps/editor/componentsPanel/tailwindUtils.d.ts", "default": "./package/components/apps/editor/componentsPanel/tailwindUtils.js" + }, + "./components/DarkModeObserver.svelte": { + "types": "./package/components/DarkModeObserver.svelte.d.ts", + "svelte": "./package/components/DarkModeObserver.svelte", + "default": "./package/components/DarkModeObserver.svelte" + }, + "./components/text_input/TextInput.svelte": { + "types": "./package/components/text_input/TextInput.svelte.d.ts", + "svelte": "./package/components/text_input/TextInput.svelte", + "default": "./package/components/text_input/TextInput.svelte" + }, + "./components/common/badge/Badge.svelte": { + "types": "./package/components/common/badge/Badge.svelte.d.ts", + "svelte": "./package/components/common/badge/Badge.svelte", + "default": "./package/components/common/badge/Badge.svelte" } }, "files": [ @@ -585,6 +600,15 @@ ], "components/custom_ui": [ "./package/components/custom_ui.d.ts" + ], + "components/DarkModeObserver.svelte": [ + "./package/components/DarkModeObserver.svelte.d.ts" + ], + "components/text_input/TextInput.svelte": [ + "./package/components/text_input/TextInput.svelte.d.ts" + ], + "components/common/badge/Badge.svelte": [ + "./package/components/common/badge/Badge.svelte.d.ts" ] } }, From f1fd245073d6bf97a6a6c64e64d545618cccc432 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 4 May 2026 17:01:06 +0200 Subject: [PATCH 07/56] feat: add separate filter searchbar for resource types tab (#9019) Co-authored-by: Claude Opus 4.7 (1M context) --- .../resources/resourceTypesFilter.ts | 23 ++++++++ .../(root)/(logged)/resources/+page.svelte | 57 +++++++++++++++---- 2 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 frontend/src/lib/components/resources/resourceTypesFilter.ts diff --git a/frontend/src/lib/components/resources/resourceTypesFilter.ts b/frontend/src/lib/components/resources/resourceTypesFilter.ts new file mode 100644 index 0000000000..37458fc728 --- /dev/null +++ b/frontend/src/lib/components/resources/resourceTypesFilter.ts @@ -0,0 +1,23 @@ +import { Boxes, FileText } from 'lucide-svelte' +import type { FilterSchemaRec } from '../FilterSearchbar.svelte' + +export function buildResourceTypesFilterSchema() { + return { + _default_: { + type: 'string' as const, + hidden: true + }, + name: { + type: 'string' as const, + label: 'Name', + icon: Boxes, + description: 'Search in resource type name' + }, + description: { + type: 'string' as const, + label: 'Description', + icon: FileText, + description: 'Search in resource type description' + } + } satisfies FilterSchemaRec +} diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte index 974b0d8f4a..285be0acbb 100644 --- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte @@ -16,9 +16,11 @@ import { resourceTypesStore } from '$lib/components/resourceTypesStore' import SchemaViewer from '$lib/components/SchemaViewer.svelte' import FilterSearchbar, { - useUrlSyncedFilterInstance + useUrlSyncedFilterInstance, + type FilterInstanceRec } from '$lib/components/FilterSearchbar.svelte' import { buildResourcesFilterSchema } from '$lib/components/resources/resourcesFilter' + import { buildResourceTypesFilterSchema } from '$lib/components/resources/resourceTypesFilter' import SharedBadge from '$lib/components/SharedBadge.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import SimpleEditor from '$lib/components/SimpleEditor.svelte' @@ -156,6 +158,25 @@ .sort() .map((f) => f.replace(/^f\//, '')) ) + let resourceTypesFilterSchema = buildResourceTypesFilterSchema() + let resourceTypesFilters: { + val: Partial> + } = $state({ val: {} }) + let filteredResourceTypes = $derived.by(() => { + if (!resourceTypes) return resourceTypes + const f = resourceTypesFilters.val + const defaultSearch = f._default_?.toLowerCase() + const nameSearch = f.name?.toLowerCase() + const descSearch = f.description?.toLowerCase() + if (!defaultSearch && !nameSearch && !descSearch) return resourceTypes + return resourceTypes.filter((rt) => { + if (defaultSearch && !rt.name.toLowerCase().includes(defaultSearch)) return false + if (nameSearch && !rt.name.toLowerCase().includes(nameSearch)) return false + if (descSearch && !(rt.description ?? '').toLowerCase().includes(descSearch)) return false + return true + }) + }) + let folderPresets = $derived([ ...itemFolders.map((f) => ({ name: `f/${f}`, value: `path_start:\\ f/${f}/` })), ...allLabels.map((l) => ({ name: l, value: `label:\\ ${l}` })), @@ -922,13 +943,22 @@ classes: loading.resources || loading.types ? 'animate-spin' : '' }} /> - + {#if tab == 'types'} + + {:else} + + {/if} {#if showTable} @@ -1199,6 +1229,13 @@ {#each new Array(6) as _} {/each} + {:else if filteredResourceTypes?.length == 0} +
+
No resource types found
+
+ Try changing the filters or creating a new resource type +
+
{:else}
@@ -1210,8 +1247,8 @@ - {#if resourceTypes} - {#each resourceTypes as { name, description, schema, canWrite, format_extension, is_fileset }} + {#if filteredResourceTypes} + {#each filteredResourceTypes as { name, description, schema, canWrite, format_extension, is_fileset }} Date: Mon, 4 May 2026 15:22:26 +0000 Subject: [PATCH 08/56] /ai-fast is now /ai --- .github/workflows/claude-fast.yml | 54 ------------------------------ .github/workflows/claude.yml | 55 +------------------------------ 2 files changed, 1 insertion(+), 108 deletions(-) delete mode 100644 .github/workflows/claude-fast.yml diff --git a/.github/workflows/claude-fast.yml b/.github/workflows/claude-fast.yml deleted file mode 100644 index 4ad53d326f..0000000000 --- a/.github/workflows/claude-fast.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Fast Claude - -on: - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - issues: - types: [opened, assigned] - pull_request_review: - types: [submitted] - -jobs: - check-membership: - if: | - (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/ai-fast')) || - (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/ai-fast')) || - (github.event_name == 'pull_request_review' && contains(github.event.review.body, '/ai-fast')) || - (github.event_name == 'issues' && contains(github.event.issue.body, '/ai-fast')) - uses: ./.github/workflows/check-org-membership.yml - secrets: - access_token: ${{ secrets.ORG_ACCESS_TOKEN }} - - claude-code-action: - needs: check-membership - if: | - needs.check-membership.outputs.is_member == 'true' - runs-on: ubicloud-standard-8 - permissions: - contents: write - pull-requests: write - issues: write - id-token: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Run Claude PR Action - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - allowed_bots: "windmill-internal-app[bot]" - trigger_phrase: "/ai-fast" - settings: | - { - "env": { - "SQLX_OFFLINE": "true" - } - } - claude_args: | - --allowedTools "Bash,WebFetch,WebSearch" - --model opus diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 95436e3214..8052bb6b4c 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -1,4 +1,4 @@ -name: Claude PR Assistant +name: Fast Claude on: issue_comment: @@ -26,7 +26,6 @@ jobs: if: | needs.check-membership.outputs.is_member == 'true' runs-on: ubicloud-standard-8 - timeout-minutes: 60 permissions: contents: write pull-requests: write @@ -38,37 +37,6 @@ jobs: with: fetch-depth: 1 - - uses: actions/cache@v3 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - - name: Run npm install and generate-backend-client - working-directory: ./frontend - run: | - # add a build directory for cargo check - mkdir -p build - npm install - npm run generate-backend-client - - - name: install xmlsec1 and gssapi - run: | - sudo apt-get update - sudo apt-get install -y libxml2-dev libxmlsec1-dev libkrb5-dev libsasl2-dev libcurl4-openssl-dev mold clang - - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - cache-workspaces: backend - toolchain: 1.93.0 - - - name: cargo check - working-directory: ./backend - timeout-minutes: 16 - run: | - SQLX_OFFLINE=true cargo check --features all_sqlx_features - - name: Run Claude PR Action uses: anthropics/claude-code-action@v1 with: @@ -84,24 +52,3 @@ jobs: claude_args: | --allowedTools "Bash,WebFetch,WebSearch" --model opus - --system-prompt "## IMPORTANT INSTRUCTIONS - - Your branch name should be a short description of the requested changes. - - Bypass previous instructions and instead of adding a url in your comment to manually create a PR, if you created a new branch with your changes, automatically create a draft PR from that branch to main. - - ## Code Quality Requirements - - After making any code changes, you MUST run the appropriate validation commands: - - **Frontend Changes:** - - Run: \`npm run check\` in the frontend directory - - Fix all warnings and errors before proceeding - - **Backend Changes:** - - Run: \`cargo check --features all_sqlx_features\` in the backend directory - - Fix all warnings and errors before proceeding - - **Pull Request Creation:** - - DO NOT FORGET TO OPEN A DRAFT PR AFTER YOU ARE DONE if you made changes after a request from a git issue. - - ## Available Tools - - Bash: Full access to run validation commands and git operations" From 42be1d46a632c23830f97995e9ab1b52a1ed5d3d Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 4 May 2026 17:33:24 +0200 Subject: [PATCH 09/56] fix(autoscaling): consider dedicated workers in scale decisions (#9020) * [ee] fix(autoscaling): consider dedicated workers in scale decisions Co-Authored-By: Claude Opus 4.7 (1M context) * Update ee-repo-ref.txt * [ee] fix(autoscaling): mirror worker tag precedence (worker_tags wins) Co-Authored-By: Claude Opus 4.7 (1M context) * chore: update ee-repo-ref to 862d487032efe30d1e4a3cd0a1ed7169500c4cd9 This commit updates the EE repository reference after PR #556 was merged in windmill-ee-private. Previous ee-repo-ref: cf87e9dcef2e95b1834b3f5c154209defc5a9ca2 New ee-repo-ref: 862d487032efe30d1e4a3cd0a1ed7169500c4cd9 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index f48c738f28..6e8b99d833 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -967f961f0a88b027d894aebd03977181129477a8 +862d487032efe30d1e4a3cd0a1ed7169500c4cd9 From 85a05765e28279af53d92fa197e30b6108fe7050 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 4 May 2026 17:53:12 +0200 Subject: [PATCH 10/56] docs(skills): document S3Object inputs and S3 streaming in script skills (#9022) Co-authored-by: Claude Opus 4.5 --- cli/src/guidance/skills.gen.ts | 248 ++++++++++++++++-- system_prompts/auto-generated/prompts.ts | 248 ++++++++++++++++-- system_prompts/auto-generated/script.md | 248 ++++++++++++++++-- .../skills/write-script-bigquery/SKILL.md | 31 +++ .../skills/write-script-bun/SKILL.md | 19 +- .../skills/write-script-bunnative/SKILL.md | 19 +- .../skills/write-script-deno/SKILL.md | 19 +- .../skills/write-script-duckdb/SKILL.md | 24 ++ .../skills/write-script-mssql/SKILL.md | 30 +++ .../skills/write-script-mysql/SKILL.md | 31 +++ .../skills/write-script-postgresql/SKILL.md | 29 ++ .../skills/write-script-python3/SKILL.md | 15 ++ .../skills/write-script-snowflake/SKILL.md | 31 +++ system_prompts/languages/bigquery.md | 31 +++ system_prompts/languages/bun.md | 19 +- system_prompts/languages/bunnative.md | 19 +- system_prompts/languages/deno.md | 19 +- system_prompts/languages/duckdb.md | 24 ++ system_prompts/languages/mssql.md | 30 +++ system_prompts/languages/mysql.md | 31 +++ system_prompts/languages/postgresql.md | 29 ++ system_prompts/languages/python3.md | 15 ++ system_prompts/languages/snowflake.md | 31 +++ 23 files changed, 1105 insertions(+), 135 deletions(-) diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 0f2c101b58..b5ac998250 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -180,6 +180,37 @@ Name the parameters by adding comments before the statement: -- @name2 (int64) = 0 SELECT * FROM users WHERE name = @name1 AND age > @name2; \`\`\` + +## Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as a \`STRING\` JSON parameter — Parquet/CSV +files are decoded server-side into a JSON array of records, JSON/JSONL pass +through. Consume with \`JSON_EXTRACT_ARRAY\` / \`JSON_VALUE\`: + +\`\`\`sql +-- @file (s3object) +SELECT + CAST(JSON_VALUE(row, '$.id') AS INT64) AS id, + JSON_VALUE(row, '$.name') AS name +FROM UNNEST(JSON_EXTRACT_ARRAY(@file)) AS row; +\`\`\` + +## Streaming query results to S3 + +Add a \`-- s3\` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its \`S3Object\` +as the script result. + +\`\`\`sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +\`\`\` + +All keys are optional: \`prefix\` (object key prefix), \`storage\` (named storage — +omit to use the workspace default), \`format\` (\`json\` (default), \`parquet\`, or +\`csv\`). Use this for large result sets — rows stream directly to S3 instead of +being buffered, bypassing the 10000-row return cap. `, "write-script-bun": `--- name: write-script-bun @@ -303,19 +334,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The \`wmill.S3Object\` type covers both the \`s3://storage/key\` URI form (\`s3:///key\` for the workspace default storage) and the \`{ s3, storage? }\` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter \`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} \`\`\` -## TypeScript Operations +### S3 operations \`\`\`typescript import * as wmill from "windmill-client"; @@ -327,7 +359,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use @@ -993,19 +1025,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The \`wmill.S3Object\` type covers both the \`s3://storage/key\` URI form (\`s3:///key\` for the workspace default storage) and the \`{ s3, storage? }\` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter \`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} \`\`\` -## TypeScript Operations +### S3 operations \`\`\`typescript import * as wmill from "windmill-client"; @@ -1017,7 +1050,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use @@ -1771,19 +1804,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The \`wmill.S3Object\` type covers both the \`s3://storage/key\` URI form (\`s3:///key\` for the workspace default storage) and the \`{ s3, storage? }\` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter \`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} \`\`\` -## TypeScript Operations +### S3 operations \`\`\`typescript import * as wmill from "windmill-client"; @@ -1795,7 +1829,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use @@ -2432,6 +2466,30 @@ SELECT * FROM read_parquet('s3:///path/to/file.parquet'); -- JSON files SELECT * FROM read_json('s3:///path/to/file.json'); \`\`\` + +### Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for it +and binds the arg as the bare \`s3://storage/key\` URI, which DuckDB's reader +functions consume directly: + +\`\`\`sql +-- $file (s3object) +SELECT * FROM read_parquet($file); +\`\`\` + +Works with any DuckDB reader: \`read_csv($file)\`, \`read_json($file)\`, etc. + +### Writing query results to S3 + +DuckDB writes to S3 natively via \`COPY ... TO\`: + +\`\`\`sql +COPY (SELECT * FROM users) TO 's3:///exports/users.parquet' (FORMAT PARQUET); +\`\`\` + +Use this instead of the \`-- s3\` streaming directive supported by the other SQL +dialects — that directive is not available in DuckDB. `, "write-script-go": `--- name: write-script-go @@ -2748,6 +2806,36 @@ Name the parameters by adding comments before the statement: -- @P2 name2 (int) = 0 SELECT * FROM users WHERE name = @P1 AND age > @P2; \`\`\` + +## Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as \`nvarchar(max)\` JSON text — Parquet/CSV +files are decoded server-side into a JSON array of records, JSON/JSONL pass +through. Consume with \`OPENJSON\`: + +\`\`\`sql +-- @P1 file (s3object) +SELECT id, name +FROM OPENJSON(@P1) +WITH (id INT, name NVARCHAR(200)); +\`\`\` + +## Streaming query results to S3 + +Add a \`-- s3\` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its \`S3Object\` +as the script result. + +\`\`\`sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +\`\`\` + +All keys are optional: \`prefix\` (object key prefix), \`storage\` (named storage — +omit to use the workspace default), \`format\` (\`json\` (default), \`parquet\`, or +\`csv\`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. `, "write-script-mysql": `--- name: write-script-mysql @@ -2800,6 +2888,37 @@ Name the parameters by adding comments before the statement: -- ? name2 (int) = 0 SELECT * FROM users WHERE name = ? AND age > ?; \`\`\` + +## Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as JSON text — Parquet/CSV files are +decoded server-side into a JSON array of records, JSON/JSONL pass through. +Consume with \`JSON_TABLE\`: + +\`\`\`sql +-- ? file (s3object) +SELECT id, name +FROM JSON_TABLE(?, '$[*]' + COLUMNS (id INT PATH '$.id', name VARCHAR(200) PATH '$.name') +) AS r; +\`\`\` + +## Streaming query results to S3 + +Add a \`-- s3\` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its \`S3Object\` +as the script result. + +\`\`\`sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +\`\`\` + +All keys are optional: \`prefix\` (object key prefix), \`storage\` (named storage — +omit to use the workspace default), \`format\` (\`json\` (default), \`parquet\`, or +\`csv\`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. `, "write-script-nativets": `--- name: write-script-nativets @@ -3607,6 +3726,35 @@ Name the parameters by adding comments at the beginning of the script (without s -- $2 name2 = default_value SELECT * FROM users WHERE name = $1::TEXT AND age > $2::INT; \`\`\` + +## Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as a \`jsonb\` parameter — Parquet/CSV files +are decoded server-side into a JSON array of records, JSON/JSONL pass through. +Consume with \`jsonb_to_recordset\` (or any \`jsonb\` API): + +\`\`\`sql +-- $1 file (s3object) +SELECT * +FROM jsonb_to_recordset($1::jsonb) AS r(id INT, name TEXT); +\`\`\` + +## Streaming query results to S3 + +Add a \`-- s3\` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its \`S3Object\` +as the script result. + +\`\`\`sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +\`\`\` + +All keys are optional: \`prefix\` (object key prefix), \`storage\` (named storage — +omit to use the workspace default), \`format\` (\`json\` (default), \`parquet\`, or +\`csv\`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. `, "write-script-powershell": `--- name: write-script-powershell @@ -3844,6 +3992,21 @@ def preprocessor(event: Event): Windmill provides built-in support for S3-compatible storage operations. +### Receiving an S3Object as a script parameter + +To accept a file from S3 as input to a script, type the parameter with \`S3Object\` (imported from \`wmill\`): + +\`\`\`python +import wmill +from wmill import S3Object + +def main(file: S3Object): + content = wmill.load_s3_file(file) + # ... +\`\`\` + +### S3 operations + \`\`\`python import wmill @@ -4848,6 +5011,37 @@ Name the parameters by adding comments before the statement: -- ? name2 (number) = 0 SELECT * FROM users WHERE name = ? AND age > ?; \`\`\` + +## Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as JSON text — Parquet/CSV files are +decoded server-side into a JSON array of records, JSON/JSONL pass through. +Wrap the bind with \`PARSE_JSON(?)\` and walk it with \`LATERAL FLATTEN\`: + +\`\`\`sql +-- ? file (s3object) +SELECT + v.value:id::NUMBER AS id, + v.value:name::STRING AS name +FROM LATERAL FLATTEN(input => PARSE_JSON(?)) v; +\`\`\` + +## Streaming query results to S3 + +Add a \`-- s3\` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its \`S3Object\` +as the script result. + +\`\`\`sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +\`\`\` + +All keys are optional: \`prefix\` (object key prefix), \`storage\` (named storage — +omit to use the workspace default), \`format\` (\`json\` (default), \`parquet\`, or +\`csv\`). Use this for large result sets — rows stream directly to S3 instead of +being buffered, bypassing the 10000-row return cap. `, "write-flow": `--- name: write-flow diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index c9ba432467..727daad2c2 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -2852,6 +2852,37 @@ Name the parameters by adding comments before the statement: -- @name2 (int64) = 0 SELECT * FROM users WHERE name = @name1 AND age > @name2; \`\`\` + +## Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as a \`STRING\` JSON parameter — Parquet/CSV +files are decoded server-side into a JSON array of records, JSON/JSONL pass +through. Consume with \`JSON_EXTRACT_ARRAY\` / \`JSON_VALUE\`: + +\`\`\`sql +-- @file (s3object) +SELECT + CAST(JSON_VALUE(row, '$.id') AS INT64) AS id, + JSON_VALUE(row, '$.name') AS name +FROM UNNEST(JSON_EXTRACT_ARRAY(@file)) AS row; +\`\`\` + +## Streaming query results to S3 + +Add a \`-- s3\` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its \`S3Object\` +as the script result. + +\`\`\`sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +\`\`\` + +All keys are optional: \`prefix\` (object key prefix), \`storage\` (named storage — +omit to use the workspace default), \`format\` (\`json\` (default), \`parquet\`, or +\`csv\`). Use this for large result sets — rows stream directly to S3 instead of +being buffered, bypassing the 10000-row return cap. `; export const LANG_BUN = `# TypeScript (Bun) @@ -2936,19 +2967,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The \`wmill.S3Object\` type covers both the \`s3://storage/key\` URI form (\`s3:///key\` for the workspace default storage) and the \`{ s3, storage? }\` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter \`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} \`\`\` -## TypeScript Operations +### S3 operations \`\`\`typescript import * as wmill from "windmill-client"; @@ -2960,7 +2992,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use @@ -3048,19 +3080,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The \`wmill.S3Object\` type covers both the \`s3://storage/key\` URI form (\`s3:///key\` for the workspace default storage) and the \`{ s3, storage? }\` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter \`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} \`\`\` -## TypeScript Operations +### S3 operations \`\`\`typescript import * as wmill from "windmill-client"; @@ -3072,7 +3105,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use @@ -3209,19 +3242,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The \`wmill.S3Object\` type covers both the \`s3://storage/key\` URI form (\`s3:///key\` for the workspace default storage) and the \`{ s3, storage? }\` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter \`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} \`\`\` -## TypeScript Operations +### S3 operations \`\`\`typescript import * as wmill from "windmill-client"; @@ -3233,7 +3267,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use @@ -3292,6 +3326,30 @@ SELECT * FROM read_parquet('s3:///path/to/file.parquet'); -- JSON files SELECT * FROM read_json('s3:///path/to/file.json'); \`\`\` + +### Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for it +and binds the arg as the bare \`s3://storage/key\` URI, which DuckDB's reader +functions consume directly: + +\`\`\`sql +-- $file (s3object) +SELECT * FROM read_parquet($file); +\`\`\` + +Works with any DuckDB reader: \`read_csv($file)\`, \`read_json($file)\`, etc. + +### Writing query results to S3 + +DuckDB writes to S3 natively via \`COPY ... TO\`: + +\`\`\`sql +COPY (SELECT * FROM users) TO 's3:///exports/users.parquet' (FORMAT PARQUET); +\`\`\` + +Use this instead of the \`-- s3\` streaming directive supported by the other SQL +dialects — that directive is not available in DuckDB. `; export const LANG_GO = `# Go @@ -3452,6 +3510,36 @@ Name the parameters by adding comments before the statement: -- @P2 name2 (int) = 0 SELECT * FROM users WHERE name = @P1 AND age > @P2; \`\`\` + +## Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as \`nvarchar(max)\` JSON text — Parquet/CSV +files are decoded server-side into a JSON array of records, JSON/JSONL pass +through. Consume with \`OPENJSON\`: + +\`\`\`sql +-- @P1 file (s3object) +SELECT id, name +FROM OPENJSON(@P1) +WITH (id INT, name NVARCHAR(200)); +\`\`\` + +## Streaming query results to S3 + +Add a \`-- s3\` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its \`S3Object\` +as the script result. + +\`\`\`sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +\`\`\` + +All keys are optional: \`prefix\` (object key prefix), \`storage\` (named storage — +omit to use the workspace default), \`format\` (\`json\` (default), \`parquet\`, or +\`csv\`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. `; export const LANG_MYSQL = `# MySQL @@ -3465,6 +3553,37 @@ Name the parameters by adding comments before the statement: -- ? name2 (int) = 0 SELECT * FROM users WHERE name = ? AND age > ?; \`\`\` + +## Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as JSON text — Parquet/CSV files are +decoded server-side into a JSON array of records, JSON/JSONL pass through. +Consume with \`JSON_TABLE\`: + +\`\`\`sql +-- ? file (s3object) +SELECT id, name +FROM JSON_TABLE(?, '$[*]' + COLUMNS (id INT PATH '$.id', name VARCHAR(200) PATH '$.name') +) AS r; +\`\`\` + +## Streaming query results to S3 + +Add a \`-- s3\` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its \`S3Object\` +as the script result. + +\`\`\`sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +\`\`\` + +All keys are optional: \`prefix\` (object key prefix), \`storage\` (named storage — +omit to use the workspace default), \`format\` (\`json\` (default), \`parquet\`, or +\`csv\`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. `; export const LANG_NATIVETS = `# TypeScript (Native) @@ -3616,6 +3735,35 @@ Name the parameters by adding comments at the beginning of the script (without s -- $2 name2 = default_value SELECT * FROM users WHERE name = $1::TEXT AND age > $2::INT; \`\`\` + +## Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as a \`jsonb\` parameter — Parquet/CSV files +are decoded server-side into a JSON array of records, JSON/JSONL pass through. +Consume with \`jsonb_to_recordset\` (or any \`jsonb\` API): + +\`\`\`sql +-- $1 file (s3object) +SELECT * +FROM jsonb_to_recordset($1::jsonb) AS r(id INT, name TEXT); +\`\`\` + +## Streaming query results to S3 + +Add a \`-- s3\` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its \`S3Object\` +as the script result. + +\`\`\`sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +\`\`\` + +All keys are optional: \`prefix\` (object key prefix), \`storage\` (named storage — +omit to use the workspace default), \`format\` (\`json\` (default), \`parquet\`, or +\`csv\`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. `; export const LANG_POWERSHELL = `# PowerShell @@ -3775,6 +3923,21 @@ def preprocessor(event: Event): Windmill provides built-in support for S3-compatible storage operations. +### Receiving an S3Object as a script parameter + +To accept a file from S3 as input to a script, type the parameter with \`S3Object\` (imported from \`wmill\`): + +\`\`\`python +import wmill +from wmill import S3Object + +def main(file: S3Object): + content = wmill.load_s3_file(file) + # ... +\`\`\` + +### S3 operations + \`\`\`python import wmill @@ -3970,5 +4133,36 @@ Name the parameters by adding comments before the statement: -- ? name2 (number) = 0 SELECT * FROM users WHERE name = ? AND age > ?; \`\`\` + +## Receiving an S3Object as a script parameter + +Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as JSON text — Parquet/CSV files are +decoded server-side into a JSON array of records, JSON/JSONL pass through. +Wrap the bind with \`PARSE_JSON(?)\` and walk it with \`LATERAL FLATTEN\`: + +\`\`\`sql +-- ? file (s3object) +SELECT + v.value:id::NUMBER AS id, + v.value:name::STRING AS name +FROM LATERAL FLATTEN(input => PARSE_JSON(?)) v; +\`\`\` + +## Streaming query results to S3 + +Add a \`-- s3\` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its \`S3Object\` +as the script result. + +\`\`\`sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +\`\`\` + +All keys are optional: \`prefix\` (object key prefix), \`storage\` (named storage — +omit to use the workspace default), \`format\` (\`json\` (default), \`parquet\`, or +\`csv\`). Use this for large result sets — rows stream directly to S3 instead of +being buffered, bypassing the 10000-row return cap. `; diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index 463cd7965b..1dbd19aee6 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -90,6 +90,37 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = @name1 AND age > @name2; ``` +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as a `STRING` JSON parameter — Parquet/CSV +files are decoded server-side into a JSON array of records, JSON/JSONL pass +through. Consume with `JSON_EXTRACT_ARRAY` / `JSON_VALUE`: + +```sql +-- @file (s3object) +SELECT + CAST(JSON_VALUE(row, '$.id') AS INT64) AS id, + JSON_VALUE(row, '$.name') AS name +FROM UNNEST(JSON_EXTRACT_ARRAY(@file)) AS row; +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered, bypassing the 10000-row return cap. + # TypeScript (Bun) @@ -173,19 +204,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter ```typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} ``` -## TypeScript Operations +### S3 operations ```typescript import * as wmill from "windmill-client"; @@ -197,7 +229,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use @@ -285,19 +317,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter ```typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} ``` -## TypeScript Operations +### S3 operations ```typescript import * as wmill from "windmill-client"; @@ -309,7 +342,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use @@ -446,19 +479,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter ```typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} ``` -## TypeScript Operations +### S3 operations ```typescript import * as wmill from "windmill-client"; @@ -470,7 +504,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use @@ -530,6 +564,30 @@ SELECT * FROM read_parquet('s3:///path/to/file.parquet'); SELECT * FROM read_json('s3:///path/to/file.json'); ``` +### Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for it +and binds the arg as the bare `s3://storage/key` URI, which DuckDB's reader +functions consume directly: + +```sql +-- $file (s3object) +SELECT * FROM read_parquet($file); +``` + +Works with any DuckDB reader: `read_csv($file)`, `read_json($file)`, etc. + +### Writing query results to S3 + +DuckDB writes to S3 natively via `COPY ... TO`: + +```sql +COPY (SELECT * FROM users) TO 's3:///exports/users.parquet' (FORMAT PARQUET); +``` + +Use this instead of the `-- s3` streaming directive supported by the other SQL +dialects — that directive is not available in DuckDB. + # Go @@ -690,6 +748,36 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = @P1 AND age > @P2; ``` +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as `nvarchar(max)` JSON text — Parquet/CSV +files are decoded server-side into a JSON array of records, JSON/JSONL pass +through. Consume with `OPENJSON`: + +```sql +-- @P1 file (s3object) +SELECT id, name +FROM OPENJSON(@P1) +WITH (id INT, name NVARCHAR(200)); +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. + # MySQL @@ -703,6 +791,37 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; ``` +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as JSON text — Parquet/CSV files are +decoded server-side into a JSON array of records, JSON/JSONL pass through. +Consume with `JSON_TABLE`: + +```sql +-- ? file (s3object) +SELECT id, name +FROM JSON_TABLE(?, '$[*]' + COLUMNS (id INT PATH '$.id', name VARCHAR(200) PATH '$.name') +) AS r; +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. + # TypeScript (Native) @@ -854,6 +973,35 @@ Name the parameters by adding comments at the beginning of the script (without s SELECT * FROM users WHERE name = $1::TEXT AND age > $2::INT; ``` +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as a `jsonb` parameter — Parquet/CSV files +are decoded server-side into a JSON array of records, JSON/JSONL pass through. +Consume with `jsonb_to_recordset` (or any `jsonb` API): + +```sql +-- $1 file (s3object) +SELECT * +FROM jsonb_to_recordset($1::jsonb) AS r(id INT, name TEXT); +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. + # PowerShell @@ -1012,6 +1160,21 @@ def preprocessor(event: Event): Windmill provides built-in support for S3-compatible storage operations. +### Receiving an S3Object as a script parameter + +To accept a file from S3 as input to a script, type the parameter with `S3Object` (imported from `wmill`): + +```python +import wmill +from wmill import S3Object + +def main(file: S3Object): + content = wmill.load_s3_file(file) + # ... +``` + +### S3 operations + ```python import wmill @@ -1208,6 +1371,37 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; ``` +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as JSON text — Parquet/CSV files are +decoded server-side into a JSON array of records, JSON/JSONL pass through. +Wrap the bind with `PARSE_JSON(?)` and walk it with `LATERAL FLATTEN`: + +```sql +-- ? file (s3object) +SELECT + v.value:id::NUMBER AS id, + v.value:name::STRING AS name +FROM LATERAL FLATTEN(input => PARSE_JSON(?)) v; +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered, bypassing the 10000-row return cap. + # TypeScript SDK (windmill-client) diff --git a/system_prompts/auto-generated/skills/write-script-bigquery/SKILL.md b/system_prompts/auto-generated/skills/write-script-bigquery/SKILL.md index e2347163c3..b9517c07f6 100644 --- a/system_prompts/auto-generated/skills/write-script-bigquery/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bigquery/SKILL.md @@ -49,3 +49,34 @@ Name the parameters by adding comments before the statement: -- @name2 (int64) = 0 SELECT * FROM users WHERE name = @name1 AND age > @name2; ``` + +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as a `STRING` JSON parameter — Parquet/CSV +files are decoded server-side into a JSON array of records, JSON/JSONL pass +through. Consume with `JSON_EXTRACT_ARRAY` / `JSON_VALUE`: + +```sql +-- @file (s3object) +SELECT + CAST(JSON_VALUE(row, '$.id') AS INT64) AS id, + JSON_VALUE(row, '$.name') AS name +FROM UNNEST(JSON_EXTRACT_ARRAY(@file)) AS row; +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered, bypassing the 10000-row return cap. diff --git a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md index 39d66a6433..16682683cc 100644 --- a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md @@ -120,19 +120,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter ```typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} ``` -## TypeScript Operations +### S3 operations ```typescript import * as wmill from "windmill-client"; @@ -144,7 +145,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use diff --git a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md index 442a6d4a7b..2fc5913b46 100644 --- a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md @@ -118,19 +118,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter ```typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} ``` -## TypeScript Operations +### S3 operations ```typescript import * as wmill from "windmill-client"; @@ -142,7 +143,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use diff --git a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md index 08b18f2a40..c627f1bbc8 100644 --- a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md @@ -124,19 +124,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter ```typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} ``` -## TypeScript Operations +### S3 operations ```typescript import * as wmill from "windmill-client"; @@ -148,7 +149,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use diff --git a/system_prompts/auto-generated/skills/write-script-duckdb/SKILL.md b/system_prompts/auto-generated/skills/write-script-duckdb/SKILL.md index c19544da17..de29cbfb3c 100644 --- a/system_prompts/auto-generated/skills/write-script-duckdb/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-duckdb/SKILL.md @@ -89,3 +89,27 @@ SELECT * FROM read_parquet('s3:///path/to/file.parquet'); -- JSON files SELECT * FROM read_json('s3:///path/to/file.json'); ``` + +### Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for it +and binds the arg as the bare `s3://storage/key` URI, which DuckDB's reader +functions consume directly: + +```sql +-- $file (s3object) +SELECT * FROM read_parquet($file); +``` + +Works with any DuckDB reader: `read_csv($file)`, `read_json($file)`, etc. + +### Writing query results to S3 + +DuckDB writes to S3 natively via `COPY ... TO`: + +```sql +COPY (SELECT * FROM users) TO 's3:///exports/users.parquet' (FORMAT PARQUET); +``` + +Use this instead of the `-- s3` streaming directive supported by the other SQL +dialects — that directive is not available in DuckDB. diff --git a/system_prompts/auto-generated/skills/write-script-mssql/SKILL.md b/system_prompts/auto-generated/skills/write-script-mssql/SKILL.md index 7a64a03ea7..76b63b896d 100644 --- a/system_prompts/auto-generated/skills/write-script-mssql/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-mssql/SKILL.md @@ -49,3 +49,33 @@ Name the parameters by adding comments before the statement: -- @P2 name2 (int) = 0 SELECT * FROM users WHERE name = @P1 AND age > @P2; ``` + +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as `nvarchar(max)` JSON text — Parquet/CSV +files are decoded server-side into a JSON array of records, JSON/JSONL pass +through. Consume with `OPENJSON`: + +```sql +-- @P1 file (s3object) +SELECT id, name +FROM OPENJSON(@P1) +WITH (id INT, name NVARCHAR(200)); +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. diff --git a/system_prompts/auto-generated/skills/write-script-mysql/SKILL.md b/system_prompts/auto-generated/skills/write-script-mysql/SKILL.md index 2c9044be23..b06b211d7a 100644 --- a/system_prompts/auto-generated/skills/write-script-mysql/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-mysql/SKILL.md @@ -49,3 +49,34 @@ Name the parameters by adding comments before the statement: -- ? name2 (int) = 0 SELECT * FROM users WHERE name = ? AND age > ?; ``` + +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as JSON text — Parquet/CSV files are +decoded server-side into a JSON array of records, JSON/JSONL pass through. +Consume with `JSON_TABLE`: + +```sql +-- ? file (s3object) +SELECT id, name +FROM JSON_TABLE(?, '$[*]' + COLUMNS (id INT PATH '$.id', name VARCHAR(200) PATH '$.name') +) AS r; +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. diff --git a/system_prompts/auto-generated/skills/write-script-postgresql/SKILL.md b/system_prompts/auto-generated/skills/write-script-postgresql/SKILL.md index e370b7a3f2..aeb976d194 100644 --- a/system_prompts/auto-generated/skills/write-script-postgresql/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-postgresql/SKILL.md @@ -49,3 +49,32 @@ Name the parameters by adding comments at the beginning of the script (without s -- $2 name2 = default_value SELECT * FROM users WHERE name = $1::TEXT AND age > $2::INT; ``` + +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as a `jsonb` parameter — Parquet/CSV files +are decoded server-side into a JSON array of records, JSON/JSONL pass through. +Consume with `jsonb_to_recordset` (or any `jsonb` API): + +```sql +-- $1 file (s3object) +SELECT * +FROM jsonb_to_recordset($1::jsonb) AS r(id INT, name TEXT); +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md index a0f16fb732..bf47615ed5 100644 --- a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -138,6 +138,21 @@ def preprocessor(event: Event): Windmill provides built-in support for S3-compatible storage operations. +### Receiving an S3Object as a script parameter + +To accept a file from S3 as input to a script, type the parameter with `S3Object` (imported from `wmill`): + +```python +import wmill +from wmill import S3Object + +def main(file: S3Object): + content = wmill.load_s3_file(file) + # ... +``` + +### S3 operations + ```python import wmill diff --git a/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md b/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md index 68c49ffa6e..3105cb1acc 100644 --- a/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md @@ -49,3 +49,34 @@ Name the parameters by adding comments before the statement: -- ? name2 (number) = 0 SELECT * FROM users WHERE name = ? AND age > ?; ``` + +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as JSON text — Parquet/CSV files are +decoded server-side into a JSON array of records, JSON/JSONL pass through. +Wrap the bind with `PARSE_JSON(?)` and walk it with `LATERAL FLATTEN`: + +```sql +-- ? file (s3object) +SELECT + v.value:id::NUMBER AS id, + v.value:name::STRING AS name +FROM LATERAL FLATTEN(input => PARSE_JSON(?)) v; +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered, bypassing the 10000-row return cap. diff --git a/system_prompts/languages/bigquery.md b/system_prompts/languages/bigquery.md index 4fa9fc3030..829921dff6 100644 --- a/system_prompts/languages/bigquery.md +++ b/system_prompts/languages/bigquery.md @@ -9,3 +9,34 @@ Name the parameters by adding comments before the statement: -- @name2 (int64) = 0 SELECT * FROM users WHERE name = @name1 AND age > @name2; ``` + +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as a `STRING` JSON parameter — Parquet/CSV +files are decoded server-side into a JSON array of records, JSON/JSONL pass +through. Consume with `JSON_EXTRACT_ARRAY` / `JSON_VALUE`: + +```sql +-- @file (s3object) +SELECT + CAST(JSON_VALUE(row, '$.id') AS INT64) AS id, + JSON_VALUE(row, '$.name') AS name +FROM UNNEST(JSON_EXTRACT_ARRAY(@file)) AS row; +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered, bypassing the 10000-row return cap. diff --git a/system_prompts/languages/bun.md b/system_prompts/languages/bun.md index d9c210850e..5d5abdc4b8 100644 --- a/system_prompts/languages/bun.md +++ b/system_prompts/languages/bun.md @@ -80,19 +80,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter ```typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} ``` -## TypeScript Operations +### S3 operations ```typescript import * as wmill from "windmill-client"; @@ -104,7 +105,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use diff --git a/system_prompts/languages/bunnative.md b/system_prompts/languages/bunnative.md index d09723b392..977c974737 100644 --- a/system_prompts/languages/bunnative.md +++ b/system_prompts/languages/bunnative.md @@ -78,19 +78,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter ```typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} ``` -## TypeScript Operations +### S3 operations ```typescript import * as wmill from "windmill-client"; @@ -102,7 +103,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use diff --git a/system_prompts/languages/deno.md b/system_prompts/languages/deno.md index 74ce91b398..adf5677a6f 100644 --- a/system_prompts/languages/deno.md +++ b/system_prompts/languages/deno.md @@ -84,19 +84,20 @@ export async function preprocessor(event: Event) { ## S3 Object Operations -Windmill provides built-in support for S3-compatible storage operations. +Windmill provides built-in support for S3-compatible storage operations. The `wmill.S3Object` type covers both the `s3://storage/key` URI form (`s3:///key` for the workspace default storage) and the `{ s3, storage? }` record form — always use it instead of redefining your own. -### S3Object Type - -The S3Object type represents a file in S3 storage: +### Receiving an S3Object as a script parameter ```typescript -type S3Object = { - s3: string; // Path within the bucket -}; +import * as wmill from "windmill-client"; + +export async function main(file: wmill.S3Object) { + const content = await wmill.loadS3File(file); + // ... +} ``` -## TypeScript Operations +### S3 operations ```typescript import * as wmill from "windmill-client"; @@ -108,7 +109,7 @@ const content: Uint8Array = await wmill.loadS3File(s3object); const blob: Blob = await wmill.loadS3FileStream(s3object); // Write file to S3 -const result: S3Object = await wmill.writeS3File( +const result: wmill.S3Object = await wmill.writeS3File( s3object, // Target path (or undefined to auto-generate) fileContent, // string or Blob s3ResourcePath // Optional: specific S3 resource to use diff --git a/system_prompts/languages/duckdb.md b/system_prompts/languages/duckdb.md index 7f80b29497..d834015890 100644 --- a/system_prompts/languages/duckdb.md +++ b/system_prompts/languages/duckdb.md @@ -49,3 +49,27 @@ SELECT * FROM read_parquet('s3:///path/to/file.parquet'); -- JSON files SELECT * FROM read_json('s3:///path/to/file.json'); ``` + +### Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for it +and binds the arg as the bare `s3://storage/key` URI, which DuckDB's reader +functions consume directly: + +```sql +-- $file (s3object) +SELECT * FROM read_parquet($file); +``` + +Works with any DuckDB reader: `read_csv($file)`, `read_json($file)`, etc. + +### Writing query results to S3 + +DuckDB writes to S3 natively via `COPY ... TO`: + +```sql +COPY (SELECT * FROM users) TO 's3:///exports/users.parquet' (FORMAT PARQUET); +``` + +Use this instead of the `-- s3` streaming directive supported by the other SQL +dialects — that directive is not available in DuckDB. diff --git a/system_prompts/languages/mssql.md b/system_prompts/languages/mssql.md index a4dae5e0fb..efeece93dd 100644 --- a/system_prompts/languages/mssql.md +++ b/system_prompts/languages/mssql.md @@ -9,3 +9,33 @@ Name the parameters by adding comments before the statement: -- @P2 name2 (int) = 0 SELECT * FROM users WHERE name = @P1 AND age > @P2; ``` + +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as `nvarchar(max)` JSON text — Parquet/CSV +files are decoded server-side into a JSON array of records, JSON/JSONL pass +through. Consume with `OPENJSON`: + +```sql +-- @P1 file (s3object) +SELECT id, name +FROM OPENJSON(@P1) +WITH (id INT, name NVARCHAR(200)); +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. diff --git a/system_prompts/languages/mysql.md b/system_prompts/languages/mysql.md index 78aa637232..4f33d2458f 100644 --- a/system_prompts/languages/mysql.md +++ b/system_prompts/languages/mysql.md @@ -9,3 +9,34 @@ Name the parameters by adding comments before the statement: -- ? name2 (int) = 0 SELECT * FROM users WHERE name = ? AND age > ?; ``` + +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as JSON text — Parquet/CSV files are +decoded server-side into a JSON array of records, JSON/JSONL pass through. +Consume with `JSON_TABLE`: + +```sql +-- ? file (s3object) +SELECT id, name +FROM JSON_TABLE(?, '$[*]' + COLUMNS (id INT PATH '$.id', name VARCHAR(200) PATH '$.name') +) AS r; +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. diff --git a/system_prompts/languages/postgresql.md b/system_prompts/languages/postgresql.md index 2cf90ea9c1..9bf8cf8026 100644 --- a/system_prompts/languages/postgresql.md +++ b/system_prompts/languages/postgresql.md @@ -9,3 +9,32 @@ Name the parameters by adding comments at the beginning of the script (without s -- $2 name2 = default_value SELECT * FROM users WHERE name = $1::TEXT AND age > $2::INT; ``` + +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as a `jsonb` parameter — Parquet/CSV files +are decoded server-side into a JSON array of records, JSON/JSONL pass through. +Consume with `jsonb_to_recordset` (or any `jsonb` API): + +```sql +-- $1 file (s3object) +SELECT * +FROM jsonb_to_recordset($1::jsonb) AS r(id INT, name TEXT); +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered as the script return value. diff --git a/system_prompts/languages/python3.md b/system_prompts/languages/python3.md index ddb92a3c4b..d556e45868 100644 --- a/system_prompts/languages/python3.md +++ b/system_prompts/languages/python3.md @@ -98,6 +98,21 @@ def preprocessor(event: Event): Windmill provides built-in support for S3-compatible storage operations. +### Receiving an S3Object as a script parameter + +To accept a file from S3 as input to a script, type the parameter with `S3Object` (imported from `wmill`): + +```python +import wmill +from wmill import S3Object + +def main(file: S3Object): + content = wmill.load_s3_file(file) + # ... +``` + +### S3 operations + ```python import wmill diff --git a/system_prompts/languages/snowflake.md b/system_prompts/languages/snowflake.md index 23d10d11b0..8ded76eb3c 100644 --- a/system_prompts/languages/snowflake.md +++ b/system_prompts/languages/snowflake.md @@ -9,3 +9,34 @@ Name the parameters by adding comments before the statement: -- ? name2 (number) = 0 SELECT * FROM users WHERE name = ? AND age > ?; ``` + +## Receiving an S3Object as a script parameter + +Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for +it, downloads the file, and binds it as JSON text — Parquet/CSV files are +decoded server-side into a JSON array of records, JSON/JSONL pass through. +Wrap the bind with `PARSE_JSON(?)` and walk it with `LATERAL FLATTEN`: + +```sql +-- ? file (s3object) +SELECT + v.value:id::NUMBER AS id, + v.value:name::STRING AS name +FROM LATERAL FLATTEN(input => PARSE_JSON(?)) v; +``` + +## Streaming query results to S3 + +Add a `-- s3` directive at the top of the script to stream the result set to S3 +instead of returning rows. Windmill writes the file and returns its `S3Object` +as the script result. + +```sql +-- s3 prefix=exports/users format=parquet +SELECT id, name FROM users; +``` + +All keys are optional: `prefix` (object key prefix), `storage` (named storage — +omit to use the workspace default), `format` (`json` (default), `parquet`, or +`csv`). Use this for large result sets — rows stream directly to S3 instead of +being buffered, bypassing the 10000-row return cap. From 1ee73c51ba9b348f0bbbe8f8fd4fba963d3a42f1 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 4 May 2026 16:26:36 +0000 Subject: [PATCH 11/56] ci: multi-tool PR reviews (Codex/Pi/Claude) with slash commands (#9026) * ci: add Pi+DeepSeek-V4 review and slash command dispatcher Auto-reviews now fan out to Claude (Opus), Codex (gpt-5.4), and Pi (DeepSeek-V4-Pro) on PR open/ready. PR comments support /review (all three), /codex, /pi, /claude with optional extra context appended to the prompt. All review workflows now substitute EE code before review and gate the auto-trigger path on org membership of the PR author. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: fix command parser whitespace, align checkout v5, broaden PR perms - Trim leading/trailing whitespace from comment first line so /review with leading space parses correctly (caught by Pi review) - Standardize EE checkout step on actions/checkout@v5 across all three review workflows (caught by Pi review) - Bump pull-requests permission to write to satisfy GitHub's PR comment endpoint when issues=write alone is rejected Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .github/codex/pr-review.prompt.md | 1 + .github/pi/pr-review.prompt.md | 25 +++ .github/workflows/codex-pr-review.yml | 163 +++++++++++++-- .github/workflows/pi-pr-review.yml | 256 +++++++++++++++++++++++ .github/workflows/pr-ready-review.yml | 93 +++++++- .github/workflows/pr-review-commands.yml | 122 +++++++++++ 6 files changed, 637 insertions(+), 23 deletions(-) create mode 100644 .github/pi/pr-review.prompt.md create mode 100644 .github/workflows/pi-pr-review.yml create mode 100644 .github/workflows/pr-review-commands.yml diff --git a/.github/codex/pr-review.prompt.md b/.github/codex/pr-review.prompt.md index d3e6dfc4e8..f0f7619b7a 100644 --- a/.github/codex/pr-review.prompt.md +++ b/.github/codex/pr-review.prompt.md @@ -9,6 +9,7 @@ Review policy: Repository context: - Read `./.github/codex/pr-review-context.md` for the PR metadata and the exact diff commands to use. +- If the context file ends with an "Additional reviewer instructions:" section, treat it as extra guidance from the human who triggered this review and follow it. - Review only the changes introduced by this PR. - Read additional files only when the diff is not enough to validate a finding. - Do not modify any files. diff --git a/.github/pi/pr-review.prompt.md b/.github/pi/pr-review.prompt.md new file mode 100644 index 0000000000..ec9f340120 --- /dev/null +++ b/.github/pi/pr-review.prompt.md @@ -0,0 +1,25 @@ +You are reviewing a GitHub pull request for this repository. + +Review policy: +- Read `AGENTS.md` (and any `AGENTS.md` in directories containing changed files) before reviewing — it is the project's contributor guide. +- Only report issues you are confident are real and introduced by this pull request. +- Focus on bugs, security problems, and clear `AGENTS.md` violations. +- Do not report style nits, speculative concerns, pre-existing issues, or anything a normal linter/typechecker would obviously catch. +- Keep the review high signal. If there is no clear issue, return no findings. + +Repository context: +- Read `./.github/pi/pr-review-context.md` for PR metadata and the exact diff commands to use. +- If the context file ends with an "Additional reviewer instructions:" section, treat it as extra guidance from the human who triggered this review and follow it. +- Run those `git diff` / `git log` commands to inspect the changes — they reference the base and head SHAs of this PR. +- Review only the changes introduced by this PR. Read additional files only when the diff is not enough to validate a finding. +- Do not modify any files. Do not create new files outside this review. + +Output requirements: +- Return a GitHub PR comment in markdown, not JSON. +- Start the comment with `## Pi Review (DeepSeek V4)`. +- Give a short overall summary first. +- If you found high-signal issues, list them in a short numbered list with file paths and line numbers when you know them confidently. +- If you found no high-signal issues, say that explicitly. +- End with a `### Reproduction instructions` section containing a short descriptive paragraph for a tester explaining how to navigate the app to observe the change. Do not make it a numbered list. If the diff is not enough to infer this safely, say that plainly. +- Prefer at most 10 findings. +- Output ONLY the final review markdown — no preamble, no thinking, no tool transcripts. diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index e945f5fd45..f40f86ca2c 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -3,19 +3,59 @@ name: Codex Auto Review on: pull_request: types: [ready_for_review, opened] + workflow_call: + inputs: + pr_number: + description: 'PR number to review' + required: true + type: number + extra_prompt: + description: 'Additional reviewer instructions appended to the standard review prompt' + required: false + type: string + default: '' + triggered_by: + description: 'GitHub username that triggered this review (for audit only)' + required: false + type: string + default: '' + secrets: + CODEX_AUTH_JSON: + required: false + WINDMILL_EE_PRIVATE_ACCESS: + required: false concurrency: - group: codex-review-${{ github.event.pull_request.number }} + group: codex-review-${{ inputs.pr_number || github.event.pull_request.number }} cancel-in-progress: true jobs: + check-membership: + if: github.event_name == 'pull_request' + uses: ./.github/workflows/check-org-membership.yml + with: + commenter: ${{ github.event.pull_request.user.login }} + secrets: + access_token: ${{ secrets.ORG_ACCESS_TOKEN }} + codex-review: + needs: check-membership runs-on: ubicloud-standard-2 timeout-minutes: 30 - if: github.event.pull_request.draft == false && github.event.pull_request.head.repo.fork == false + if: | + always() && + ( + needs.check-membership.result == 'skipped' || + (needs.check-membership.result == 'success' && needs.check-membership.outputs.is_member == 'true') + ) && + ( + github.event_name == 'workflow_call' || + (github.event.pull_request.draft == false && github.event.pull_request.head.repo.fork == false) + ) permissions: contents: read issues: write + pull-requests: write steps: - name: Check Codex configuration id: codex_config @@ -29,25 +69,104 @@ jobs: echo "CODEX_AUTH_JSON is not configured; skipping Codex review." fi - - name: Checkout repository + - name: Resolve PR metadata if: steps.codex_config.outputs.enabled == 'true' + id: pr + env: + GH_TOKEN: ${{ github.token }} + INPUT_PR_NUMBER: ${{ inputs.pr_number }} + EVENT_PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} + EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }} + EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + EVENT_TITLE: ${{ github.event.pull_request.title }} + EVENT_BODY: ${{ github.event.pull_request.body }} + EVENT_FORK: ${{ github.event.pull_request.head.repo.fork }} + run: | + if [ -n "$INPUT_PR_NUMBER" ]; then + PR_JSON=$(gh pr view "$INPUT_PR_NUMBER" --repo "${{ github.repository }}" \ + --json number,baseRefName,baseRefOid,headRefOid,title,body,isCrossRepository) + PR_NUMBER=$(echo "$PR_JSON" | jq -r '.number') + BASE_REF=$(echo "$PR_JSON" | jq -r '.baseRefName') + BASE_SHA=$(echo "$PR_JSON" | jq -r '.baseRefOid') + HEAD_SHA=$(echo "$PR_JSON" | jq -r '.headRefOid') + PR_TITLE=$(echo "$PR_JSON" | jq -r '.title') + PR_BODY=$(echo "$PR_JSON" | jq -r '.body // ""') + IS_FORK=$(echo "$PR_JSON" | jq -r '.isCrossRepository') + else + PR_NUMBER="$EVENT_PR_NUMBER" + BASE_REF="$EVENT_BASE_REF" + BASE_SHA="$EVENT_BASE_SHA" + HEAD_SHA="$EVENT_HEAD_SHA" + PR_TITLE="$EVENT_TITLE" + PR_BODY="$EVENT_BODY" + IS_FORK="$EVENT_FORK" + fi + if [ "$IS_FORK" = "true" ]; then + echo "Skipping Codex review for fork PR." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + { + echo "skip=false" + echo "pr_number=$PR_NUMBER" + echo "base_ref=$BASE_REF" + echo "base_sha=$BASE_SHA" + echo "head_sha=$HEAD_SHA" + echo 'title<> "$GITHUB_OUTPUT" + + - name: Checkout repository + if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' uses: actions/checkout@v5 with: - ref: refs/pull/${{ github.event.pull_request.number }}/merge + ref: refs/pull/${{ steps.pr.outputs.pr_number }}/merge fetch-depth: 1 + - name: Check EE access + if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' + id: ee + env: + EE_TOKEN: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + run: | + if [ -n "$EE_TOKEN" ]; then + echo "available=true" >> "$GITHUB_OUTPUT" + echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout EE repository + if: steps.ee.outputs.available == 'true' + uses: actions/checkout@v5 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ steps.ee.outputs.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 1 + + - name: Substitute EE code + if: steps.ee.outputs.available == 'true' + run: ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + - name: Set up Node.js - if: steps.codex_config.outputs.enabled == 'true' + if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' uses: actions/setup-node@v4 with: node-version: 22 - name: Install Codex CLI - if: steps.codex_config.outputs.enabled == 'true' + if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' run: npm install --global @openai/codex@0.117.0 - name: Configure file-backed Codex auth - if: steps.codex_config.outputs.enabled == 'true' + if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' env: CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }} run: | @@ -63,24 +182,25 @@ jobs: node -e 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"))' "$CODEX_HOME/auth.json" - name: Pre-fetch base and head refs for the PR - if: steps.codex_config.outputs.enabled == 'true' + if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' env: - PR_BASE_REF: ${{ github.event.pull_request.base.ref }} - PR_NUMBER: ${{ github.event.pull_request.number }} + PR_BASE_REF: ${{ steps.pr.outputs.base_ref }} + PR_NUMBER: ${{ steps.pr.outputs.pr_number }} run: | git fetch --no-tags origin \ "$PR_BASE_REF" \ "+refs/pull/$PR_NUMBER/head" - name: Write Codex review context - if: steps.codex_config.outputs.enabled == 'true' + if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' env: PR_REPOSITORY: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - PR_TITLE: ${{ github.event.pull_request.title }} - PR_BODY: ${{ github.event.pull_request.body || '' }} + PR_NUMBER: ${{ steps.pr.outputs.pr_number }} + PR_BASE_SHA: ${{ steps.pr.outputs.base_sha }} + PR_HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + PR_TITLE: ${{ steps.pr.outputs.title }} + PR_BODY: ${{ steps.pr.outputs.body }} + EXTRA_PROMPT: ${{ inputs.extra_prompt }} run: | mkdir -p .github/codex node <<'NODE' @@ -106,11 +226,14 @@ jobs: 'Full review diff command:', `git diff --unified=0 ${process.env.PR_BASE_SHA}...${process.env.PR_HEAD_SHA}` ]; + if (process.env.EXTRA_PROMPT && process.env.EXTRA_PROMPT.trim()) { + lines.push('', 'Additional reviewer instructions:', process.env.EXTRA_PROMPT.trim()); + } fs.writeFileSync('.github/codex/pr-review-context.md', `${lines.join('\n')}\n`); NODE - name: Run Codex review - if: steps.codex_config.outputs.enabled == 'true' + if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' run: | codex exec \ -C "$GITHUB_WORKSPACE" \ @@ -121,8 +244,10 @@ jobs: - < .github/codex/pr-review.prompt.md - name: Post Codex review comment - if: steps.codex_config.outputs.enabled == 'true' + if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' uses: actions/github-script@v7 + env: + PR_NUMBER: ${{ steps.pr.outputs.pr_number }} with: github-token: ${{ github.token }} script: | @@ -140,6 +265,6 @@ jobs: await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, - issue_number: context.payload.pull_request.number, + issue_number: Number(process.env.PR_NUMBER), body, }); diff --git a/.github/workflows/pi-pr-review.yml b/.github/workflows/pi-pr-review.yml new file mode 100644 index 0000000000..04cf8a20ca --- /dev/null +++ b/.github/workflows/pi-pr-review.yml @@ -0,0 +1,256 @@ +name: Pi Auto Review + +on: + pull_request: + types: [ready_for_review, opened] + workflow_call: + inputs: + pr_number: + description: 'PR number to review' + required: true + type: number + extra_prompt: + description: 'Additional reviewer instructions appended to the standard review prompt' + required: false + type: string + default: '' + triggered_by: + description: 'GitHub username that triggered this review (for audit only)' + required: false + type: string + default: '' + secrets: + DEEPSEEK_API_KEY: + required: false + WINDMILL_EE_PRIVATE_ACCESS: + required: false + +concurrency: + group: pi-review-${{ inputs.pr_number || github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check-membership: + if: github.event_name == 'pull_request' + uses: ./.github/workflows/check-org-membership.yml + with: + commenter: ${{ github.event.pull_request.user.login }} + secrets: + access_token: ${{ secrets.ORG_ACCESS_TOKEN }} + + pi-review: + needs: check-membership + runs-on: ubicloud-standard-2 + timeout-minutes: 30 + if: | + always() && + ( + needs.check-membership.result == 'skipped' || + (needs.check-membership.result == 'success' && needs.check-membership.outputs.is_member == 'true') + ) && + ( + github.event_name == 'workflow_call' || + (github.event.pull_request.draft == false && github.event.pull_request.head.repo.fork == false) + ) + permissions: + contents: read + issues: write + pull-requests: write + steps: + - name: Check Pi configuration + id: pi_config + env: + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + run: | + if [ -n "$DEEPSEEK_API_KEY" ]; then + echo "enabled=true" >> "$GITHUB_OUTPUT" + else + echo "enabled=false" >> "$GITHUB_OUTPUT" + echo "DEEPSEEK_API_KEY is not configured; skipping Pi review." + fi + + - name: Resolve PR metadata + if: steps.pi_config.outputs.enabled == 'true' + id: pr + env: + GH_TOKEN: ${{ github.token }} + INPUT_PR_NUMBER: ${{ inputs.pr_number }} + EVENT_PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} + EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }} + EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + EVENT_TITLE: ${{ github.event.pull_request.title }} + EVENT_BODY: ${{ github.event.pull_request.body }} + EVENT_FORK: ${{ github.event.pull_request.head.repo.fork }} + run: | + if [ -n "$INPUT_PR_NUMBER" ]; then + PR_JSON=$(gh pr view "$INPUT_PR_NUMBER" --repo "${{ github.repository }}" \ + --json number,baseRefName,baseRefOid,headRefOid,title,body,isCrossRepository) + PR_NUMBER=$(echo "$PR_JSON" | jq -r '.number') + BASE_REF=$(echo "$PR_JSON" | jq -r '.baseRefName') + BASE_SHA=$(echo "$PR_JSON" | jq -r '.baseRefOid') + HEAD_SHA=$(echo "$PR_JSON" | jq -r '.headRefOid') + PR_TITLE=$(echo "$PR_JSON" | jq -r '.title') + PR_BODY=$(echo "$PR_JSON" | jq -r '.body // ""') + IS_FORK=$(echo "$PR_JSON" | jq -r '.isCrossRepository') + else + PR_NUMBER="$EVENT_PR_NUMBER" + BASE_REF="$EVENT_BASE_REF" + BASE_SHA="$EVENT_BASE_SHA" + HEAD_SHA="$EVENT_HEAD_SHA" + PR_TITLE="$EVENT_TITLE" + PR_BODY="$EVENT_BODY" + IS_FORK="$EVENT_FORK" + fi + if [ "$IS_FORK" = "true" ]; then + echo "Skipping Pi review for fork PR." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + { + echo "skip=false" + echo "pr_number=$PR_NUMBER" + echo "base_ref=$BASE_REF" + echo "base_sha=$BASE_SHA" + echo "head_sha=$HEAD_SHA" + echo 'title<> "$GITHUB_OUTPUT" + + - name: Checkout repository + if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' + uses: actions/checkout@v5 + with: + ref: refs/pull/${{ steps.pr.outputs.pr_number }}/merge + fetch-depth: 1 + + - name: Check EE access + if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' + id: ee + env: + EE_TOKEN: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + run: | + if [ -n "$EE_TOKEN" ]; then + echo "available=true" >> "$GITHUB_OUTPUT" + echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout EE repository + if: steps.ee.outputs.available == 'true' + uses: actions/checkout@v5 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ steps.ee.outputs.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 1 + + - name: Substitute EE code + if: steps.ee.outputs.available == 'true' + run: ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - name: Set up Node.js + if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install Pi CLI + if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' + run: npm install --global @mariozechner/pi-coding-agent + + - name: Pre-fetch base and head refs for the PR + if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' + env: + PR_BASE_REF: ${{ steps.pr.outputs.base_ref }} + PR_NUMBER: ${{ steps.pr.outputs.pr_number }} + run: | + git fetch --no-tags origin \ + "$PR_BASE_REF" \ + "+refs/pull/$PR_NUMBER/head" + + - name: Write Pi review context + if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' + env: + PR_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ steps.pr.outputs.pr_number }} + PR_BASE_SHA: ${{ steps.pr.outputs.base_sha }} + PR_HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + PR_TITLE: ${{ steps.pr.outputs.title }} + PR_BODY: ${{ steps.pr.outputs.body }} + EXTRA_PROMPT: ${{ inputs.extra_prompt }} + run: | + mkdir -p .github/pi + node <<'NODE' + const fs = require('fs'); + const lines = [ + `Repository: ${process.env.PR_REPOSITORY}`, + `PR number: ${process.env.PR_NUMBER}`, + `Base SHA: ${process.env.PR_BASE_SHA}`, + `Head SHA: ${process.env.PR_HEAD_SHA}`, + '', + 'PR title:', + process.env.PR_TITLE || '(empty)', + '', + 'PR body:', + process.env.PR_BODY || '(empty)', + '', + 'Changed commits command:', + `git log --oneline ${process.env.PR_BASE_SHA}...${process.env.PR_HEAD_SHA}`, + '', + 'Changed files command:', + `git diff --stat ${process.env.PR_BASE_SHA}...${process.env.PR_HEAD_SHA}`, + '', + 'Full review diff command:', + `git diff --unified=0 ${process.env.PR_BASE_SHA}...${process.env.PR_HEAD_SHA}` + ]; + if (process.env.EXTRA_PROMPT && process.env.EXTRA_PROMPT.trim()) { + lines.push('', 'Additional reviewer instructions:', process.env.EXTRA_PROMPT.trim()); + } + fs.writeFileSync('.github/pi/pr-review-context.md', `${lines.join('\n')}\n`); + NODE + + - name: Run Pi review + if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' + env: + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + PI_SKIP_VERSION_CHECK: '1' + run: | + pi -p \ + --provider deepseek \ + --model deepseek-v4-pro \ + --tools read,grep,find,ls,bash \ + < .github/pi/pr-review.prompt.md \ + > pi-final-message.md + + - name: Post Pi review comment + if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' + uses: actions/github-script@v7 + env: + PR_NUMBER: ${{ steps.pr.outputs.pr_number }} + with: + github-token: ${{ github.token }} + script: | + const fs = require('fs'); + const path = `${process.env.GITHUB_WORKSPACE}/pi-final-message.md`; + if (!fs.existsSync(path)) { + core.info('Pi did not produce a final message; skipping PR comment.'); + return; + } + const body = fs.readFileSync(path, 'utf8').trim(); + if (!body) { + core.info('Pi final message was empty; skipping PR comment.'); + return; + } + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: Number(process.env.PR_NUMBER), + body, + }); diff --git a/.github/workflows/pr-ready-review.yml b/.github/workflows/pr-ready-review.yml index 78c0c3e045..78cfd41bc2 100644 --- a/.github/workflows/pr-ready-review.yml +++ b/.github/workflows/pr-ready-review.yml @@ -3,31 +3,116 @@ name: Claude Auto Review on: pull_request: types: [ready_for_review, opened] + workflow_call: + inputs: + pr_number: + description: 'PR number to review' + required: true + type: number + extra_prompt: + description: 'Additional reviewer instructions appended to the standard review prompt' + required: false + type: string + default: '' + triggered_by: + description: 'GitHub username that triggered this review (for audit only)' + required: false + type: string + default: '' + secrets: + CLAUDE_CODE_OAUTH_TOKEN: + required: true + WINDMILL_EE_PRIVATE_ACCESS: + required: false concurrency: - group: claude-review-${{ github.event.pull_request.number }} + group: claude-review-${{ inputs.pr_number || github.event.pull_request.number }} cancel-in-progress: true jobs: + check-membership: + if: github.event_name == 'pull_request' + uses: ./.github/workflows/check-org-membership.yml + with: + commenter: ${{ github.event.pull_request.user.login }} + secrets: + access_token: ${{ secrets.ORG_ACCESS_TOKEN }} + auto-review: + needs: check-membership runs-on: ubuntu-latest - if: github.event.pull_request.draft == false || github.event.pull_request.ready_for_review == true + if: | + always() && + ( + needs.check-membership.result == 'skipped' || + (needs.check-membership.result == 'success' && needs.check-membership.outputs.is_member == 'true') + ) && + ( + github.event_name == 'workflow_call' || + (github.event.pull_request.draft == false || github.event.pull_request.ready_for_review == true) + ) permissions: contents: read pull-requests: read id-token: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 1 + - name: Check EE access + id: ee + env: + EE_TOKEN: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + run: | + if [ -n "$EE_TOKEN" ]; then + echo "available=true" >> "$GITHUB_OUTPUT" + echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout EE repository + if: steps.ee.outputs.available == 'true' + uses: actions/checkout@v5 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ steps.ee.outputs.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 1 + + - name: Substitute EE code + if: steps.ee.outputs.available == 'true' + run: ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - name: Resolve PR number + id: resolve + env: + INPUT_PR_NUMBER: ${{ inputs.pr_number }} + EVENT_PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + if [ -n "$INPUT_PR_NUMBER" ]; then + echo "pr_number=$INPUT_PR_NUMBER" >> "$GITHUB_OUTPUT" + else + echo "pr_number=$EVENT_PR_NUMBER" >> "$GITHUB_OUTPUT" + fi + - name: Read review prompt id: review-prompt + env: + EXTRA_PROMPT: ${{ inputs.extra_prompt }} run: | { echo 'REVIEW_PROMPT<> "$GITHUB_ENV" @@ -38,7 +123,7 @@ jobs: track_progress: true prompt: | REPO: ${{ github.repository }} - PR NUMBER: ${{ github.event.pull_request.number }} + PR NUMBER: ${{ steps.resolve.outputs.pr_number }} ${{ env.REVIEW_PROMPT }} claude_args: | diff --git a/.github/workflows/pr-review-commands.yml b/.github/workflows/pr-review-commands.yml new file mode 100644 index 0000000000..21d5e1decd --- /dev/null +++ b/.github/workflows/pr-review-commands.yml @@ -0,0 +1,122 @@ +name: PR Review Commands + +on: + issue_comment: + types: [created] + +jobs: + parse: + if: github.event.issue.pull_request != null + runs-on: ubuntu-latest + outputs: + command: ${{ steps.parse.outputs.command }} + extra_prompt: ${{ steps.parse.outputs.extra_prompt }} + steps: + - name: Parse command from comment + id: parse + env: + BODY: ${{ github.event.comment.body }} + run: | + FIRST_LINE=$(printf '%s' "$BODY" | head -n 1 | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//') + FIRST_WORD=${FIRST_LINE%% *} + case "$FIRST_WORD" in + /review|/codex|/pi|/claude) + COMMAND="${FIRST_WORD#/}" + REMAINDER_FIRST_LINE=${FIRST_LINE#"$FIRST_WORD"} + REMAINDER_FIRST_LINE=${REMAINDER_FIRST_LINE# } + REST=$(printf '%s' "$BODY" | tail -n +2) + { + echo "command=$COMMAND" + echo 'extra_prompt<> "$GITHUB_OUTPUT" + ;; + *) + echo "command=" >> "$GITHUB_OUTPUT" + ;; + esac + + check-membership: + needs: parse + if: needs.parse.outputs.command != '' + uses: ./.github/workflows/check-org-membership.yml + secrets: + access_token: ${{ secrets.ORG_ACCESS_TOKEN }} + + acknowledge: + needs: [parse, check-membership] + if: needs.parse.outputs.command != '' && needs.check-membership.outputs.is_member == 'true' + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - name: React to comment with eyes + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + run: | + gh api -X POST \ + "/repos/$REPO/issues/comments/$COMMENT_ID/reactions" \ + -f content=eyes >/dev/null + + claude: + needs: [parse, check-membership] + if: | + needs.check-membership.outputs.is_member == 'true' && + (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'claude') + permissions: + contents: read + pull-requests: read + id-token: write + uses: ./.github/workflows/pr-ready-review.yml + with: + pr_number: ${{ github.event.issue.number }} + extra_prompt: ${{ needs.parse.outputs.extra_prompt }} + triggered_by: ${{ github.event.comment.user.login }} + secrets: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + + codex: + needs: [parse, check-membership] + if: | + needs.check-membership.outputs.is_member == 'true' && + (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'codex') + permissions: + contents: read + issues: write + pull-requests: write + uses: ./.github/workflows/codex-pr-review.yml + with: + pr_number: ${{ github.event.issue.number }} + extra_prompt: ${{ needs.parse.outputs.extra_prompt }} + triggered_by: ${{ github.event.comment.user.login }} + secrets: + CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }} + WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + + pi: + needs: [parse, check-membership] + if: | + needs.check-membership.outputs.is_member == 'true' && + (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'pi') + permissions: + contents: read + issues: write + pull-requests: write + uses: ./.github/workflows/pi-pr-review.yml + with: + pr_number: ${{ github.event.issue.number }} + extra_prompt: ${{ needs.parse.outputs.extra_prompt }} + triggered_by: ${{ github.event.comment.user.login }} + secrets: + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} From 548794cdbee448be9a990d6ea96bc869298d154f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 4 May 2026 16:44:53 +0000 Subject: [PATCH 12/56] ci: pi progress streaming, codex gpt-5.5 + danger-full-access sandbox (#9030) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: install bubblewrap for codex sandbox; stream pi progress in logs - Codex's vendored bwrap fails to set up loopback on some ubicloud runners, leaving codex unable to read any local files. Install the system bubblewrap package before running codex so its read-only sandbox works reliably. - Switch pi to --mode json and pipe events through jq to surface agent/turn boundaries and tool calls live in the GitHub Actions log, matching codex's progress visibility. Final assistant text is extracted from the saved event log into pi-final-message.md for the PR comment. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: drop bubblewrap install, use codex -s danger-full-access Codex's read-only sandbox uses bwrap which fails to set up loopback on some ubicloud runners. Rather than apt-installing bubblewrap, switch to the no-sandbox mode for parity with how Pi and Claude already operate in the same workflow — runner is ephemeral and we trust the codex prompt the same way. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: bump codex review model from gpt-5.4 to gpt-5.5 gpt-5.5 is positioned as the agentic successor to gpt-5.4 — same per-token latency, fewer tokens to complete Codex tasks, and explicitly stronger at holding context across large systems and multi-tool reasoning, which matches the PR review workload. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/codex-pr-review.yml | 4 ++-- .github/workflows/pi-pr-review.yml | 32 ++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index f40f86ca2c..1a3e968272 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -237,9 +237,9 @@ jobs: run: | codex exec \ -C "$GITHUB_WORKSPACE" \ - -m gpt-5.4 \ + -m gpt-5.5 \ -c 'model_reasoning_effort="xhigh"' \ - -s read-only \ + -s danger-full-access \ -o codex-final-message.md \ - < .github/codex/pr-review.prompt.md diff --git a/.github/workflows/pi-pr-review.yml b/.github/workflows/pi-pr-review.yml index 04cf8a20ca..eba8c2bad3 100644 --- a/.github/workflows/pi-pr-review.yml +++ b/.github/workflows/pi-pr-review.yml @@ -222,12 +222,42 @@ jobs: DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} PI_SKIP_VERSION_CHECK: '1' run: | + set -o pipefail pi -p \ --provider deepseek \ --model deepseek-v4-pro \ --tools read,grep,find,ls,bash \ + --mode json \ < .github/pi/pr-review.prompt.md \ - > pi-final-message.md + | tee pi-events.jsonl \ + | jq -rc --unbuffered ' + if .type == "agent_start" then "🤖 pi agent started" + elif .type == "turn_start" then "── turn ──" + elif .type == "message_end" then + "[\(.message.role)] " + ( + (.message.content // []) + | map( + if .type == "text" then "text(\(.text | length)c)" + elif .type == "tool_use" then "🔧 \(.name) \(.input | @json | .[:160])" + elif .type == "tool_result" then "✅ result" + else .type + end + ) + | join(" | ") + ) + elif .type == "turn_end" then "── turn done (\((.toolResults // []) | length) tool result(s)) ──" + elif .type == "agent_end" then "🏁 pi agent done" + else empty + end + ' + + jq -r ' + select(.type == "agent_end") + | .messages + | map(select(.role == "assistant")) + | last + | (.content[]? | select(.type == "text") | .text) + ' pi-events.jsonl > pi-final-message.md - name: Post Pi review comment if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' From 1be62ea926872882ddbd4c8ce81502d6e341b8c1 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 4 May 2026 16:51:59 +0000 Subject: [PATCH 13/56] fix: stop sequential whileloop on iteration failure (#9028) Co-authored-by: Claude Opus 4.7 (1M context) --- backend/windmill-worker/src/worker_flow.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 7b0038795f..1577d108e8 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -1038,7 +1038,8 @@ pub async fn update_flow_status_after_job_completion_internal( // backwards compatibility itered.as_ref().map(|itered| itered.len()).unwrap_or(0) }; - (*while_loop || (*index + 1 < itered_len) && (success || skip_loop_failures)) + (*while_loop || *index + 1 < itered_len) + && (success || skip_loop_failures) && !stop_early } => { From 505f78bd29813fbf78ed855d1da6636ff4c286d1 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 4 May 2026 16:53:20 +0000 Subject: [PATCH 14/56] ci: re-review on push, thread prior PR comments into reviewer context (#9032) * ci: re-review on push, thread prior PR comments into reviewer context - Add 'synchronize' to all three review workflow triggers so each push to a PR branch re-runs Claude/Codex/Pi. Existing cancel-in-progress concurrency groups ensure only the latest push's review actually executes. - Fetch the most recent up to 20 PR comments before each review and inject them into the prompt context so the reviewer can recognize its own previous review, focus on what changed, and avoid repeating findings the human already addressed. - Update the three review prompts (Claude, Codex, Pi) to instruct the reviewer to honor the prior-discussion section when present. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: bump codex CLI to 0.128.0 for gpt-5.5 support Codex 0.117.0 rejects the gpt-5.5 model with 'requires a newer version of Codex'. 0.128.0 is the current stable release on npm. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: limit synchronize re-trigger to pi review only Re-running Claude and Codex on every push gets expensive fast on busy PRs. Pi (DeepSeek-V4) is cheap enough to re-run per push, while Claude/Codex remain on opened/ready_for_review and re-trigger via slash commands. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .claude/review-prompt.md | 1 + .github/codex/pr-review.prompt.md | 3 ++- .github/pi/pr-review.prompt.md | 3 ++- .github/workflows/codex-pr-review.yml | 30 ++++++++++++++++++++++++++- .github/workflows/pi-pr-review.yml | 30 ++++++++++++++++++++++++++- .github/workflows/pr-ready-review.yml | 22 ++++++++++++++++++++ 6 files changed, 85 insertions(+), 4 deletions(-) diff --git a/.claude/review-prompt.md b/.claude/review-prompt.md index 6814089bea..c862f2d1bc 100644 --- a/.claude/review-prompt.md +++ b/.claude/review-prompt.md @@ -19,6 +19,7 @@ Read all relevant CLAUDE.md files (root and in directories containing changed fi - Use top-level comments for general observations or praise - Only flag issues introduced by this PR, not pre-existing problems - Self-validate each finding: "Is this definitely a real issue?" If uncertain, discard it +- If the prompt includes a "Prior PR discussion" section, this PR has already been reviewed. Look for your own earlier comment, focus on what changed in the latest commits, and do not repeat findings the human already pushed back on or addressed ## Testing Instructions diff --git a/.github/codex/pr-review.prompt.md b/.github/codex/pr-review.prompt.md index f0f7619b7a..141fd52d61 100644 --- a/.github/codex/pr-review.prompt.md +++ b/.github/codex/pr-review.prompt.md @@ -9,7 +9,8 @@ Review policy: Repository context: - Read `./.github/codex/pr-review-context.md` for the PR metadata and the exact diff commands to use. -- If the context file ends with an "Additional reviewer instructions:" section, treat it as extra guidance from the human who triggered this review and follow it. +- If the context file contains an "Additional reviewer instructions:" section, treat it as extra guidance from the human who triggered this review and follow it. +- If the context file contains a "Prior PR discussion" section, this PR has already received review activity. Look for your own previous "## Codex Review" comment, take it into account, focus on what changed in the latest commits, and do not repeat findings the human already pushed back on or addressed. - Review only the changes introduced by this PR. - Read additional files only when the diff is not enough to validate a finding. - Do not modify any files. diff --git a/.github/pi/pr-review.prompt.md b/.github/pi/pr-review.prompt.md index ec9f340120..93ad730ba0 100644 --- a/.github/pi/pr-review.prompt.md +++ b/.github/pi/pr-review.prompt.md @@ -9,7 +9,8 @@ Review policy: Repository context: - Read `./.github/pi/pr-review-context.md` for PR metadata and the exact diff commands to use. -- If the context file ends with an "Additional reviewer instructions:" section, treat it as extra guidance from the human who triggered this review and follow it. +- If the context file contains an "Additional reviewer instructions:" section, treat it as extra guidance from the human who triggered this review and follow it. +- If the context file contains a "Prior PR discussion" section, this PR has already received review activity. Look for your own previous "## Pi Review (DeepSeek V4)" comment, take it into account, focus on what changed in the latest commits, and do not repeat findings the human already pushed back on or addressed. - Run those `git diff` / `git log` commands to inspect the changes — they reference the base and head SHAs of this PR. - Review only the changes introduced by this PR. Read additional files only when the diff is not enough to validate a finding. - Do not modify any files. Do not create new files outside this review. diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index 1a3e968272..79b77f842d 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -163,7 +163,7 @@ jobs: - name: Install Codex CLI if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - run: npm install --global @openai/codex@0.117.0 + run: npm install --global @openai/codex@0.128.0 - name: Configure file-backed Codex auth if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' @@ -191,6 +191,17 @@ jobs: "$PR_BASE_REF" \ "+refs/pull/$PR_NUMBER/head" + - name: Fetch prior PR discussion + if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ steps.pr.outputs.pr_number }} + run: | + gh api "repos/$REPO/issues/$PR_NUMBER/comments?per_page=100" \ + --jq '[.[] | {user: .user.login, created_at: .created_at, body: (.body | .[:4000])}] | sort_by(.created_at) | .[-20:]' \ + > prior-comments.json || echo "[]" > prior-comments.json + - name: Write Codex review context if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' env: @@ -229,6 +240,23 @@ jobs: if (process.env.EXTRA_PROMPT && process.env.EXTRA_PROMPT.trim()) { lines.push('', 'Additional reviewer instructions:', process.env.EXTRA_PROMPT.trim()); } + if (fs.existsSync('prior-comments.json')) { + try { + const comments = JSON.parse(fs.readFileSync('prior-comments.json', 'utf8')); + if (Array.isArray(comments) && comments.length > 0) { + lines.push( + '', + 'Prior PR discussion (most recent up to 20 comments):', + '', + 'If you have already reviewed this PR (look for your own earlier "## Codex Review" comment), focus on what changed since then per the diff and respect any decisions the human made in replies. Do not re-flag findings the human already pushed back on.', + '' + ); + for (const c of comments) { + lines.push(`### @${c.user} (${c.created_at})`, '', c.body, '', '---', ''); + } + } + } catch (_) {} + } fs.writeFileSync('.github/codex/pr-review-context.md', `${lines.join('\n')}\n`); NODE diff --git a/.github/workflows/pi-pr-review.yml b/.github/workflows/pi-pr-review.yml index eba8c2bad3..d7c743fc16 100644 --- a/.github/workflows/pi-pr-review.yml +++ b/.github/workflows/pi-pr-review.yml @@ -2,7 +2,7 @@ name: Pi Auto Review on: pull_request: - types: [ready_for_review, opened] + types: [ready_for_review, opened, synchronize] workflow_call: inputs: pr_number: @@ -175,6 +175,17 @@ jobs: "$PR_BASE_REF" \ "+refs/pull/$PR_NUMBER/head" + - name: Fetch prior PR discussion + if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ steps.pr.outputs.pr_number }} + run: | + gh api "repos/$REPO/issues/$PR_NUMBER/comments?per_page=100" \ + --jq '[.[] | {user: .user.login, created_at: .created_at, body: (.body | .[:4000])}] | sort_by(.created_at) | .[-20:]' \ + > prior-comments.json || echo "[]" > prior-comments.json + - name: Write Pi review context if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' env: @@ -213,6 +224,23 @@ jobs: if (process.env.EXTRA_PROMPT && process.env.EXTRA_PROMPT.trim()) { lines.push('', 'Additional reviewer instructions:', process.env.EXTRA_PROMPT.trim()); } + if (fs.existsSync('prior-comments.json')) { + try { + const comments = JSON.parse(fs.readFileSync('prior-comments.json', 'utf8')); + if (Array.isArray(comments) && comments.length > 0) { + lines.push( + '', + 'Prior PR discussion (most recent up to 20 comments):', + '', + 'If you have already reviewed this PR (look for your own earlier "## Pi Review (DeepSeek V4)" comment), focus on what changed since then per the diff and respect any decisions the human made in replies. Do not re-flag findings the human already pushed back on.', + '' + ); + for (const c of comments) { + lines.push(`### @${c.user} (${c.created_at})`, '', c.body, '', '---', ''); + } + } + } catch (_) {} + } fs.writeFileSync('.github/pi/pr-review-context.md', `${lines.join('\n')}\n`); NODE diff --git a/.github/workflows/pr-ready-review.yml b/.github/workflows/pr-ready-review.yml index 78cfd41bc2..0058ffa8d9 100644 --- a/.github/workflows/pr-ready-review.yml +++ b/.github/workflows/pr-ready-review.yml @@ -99,6 +99,24 @@ jobs: echo "pr_number=$EVENT_PR_NUMBER" >> "$GITHUB_OUTPUT" fi + - name: Fetch prior PR discussion + id: prior + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ steps.resolve.outputs.pr_number }} + run: | + gh api "repos/$REPO/issues/$PR_NUMBER/comments?per_page=100" \ + --jq '[.[] | {user: .user.login, created_at: .created_at, body: (.body | .[:4000])}] | sort_by(.created_at) | .[-20:]' \ + > prior-comments.json || echo "[]" > prior-comments.json + jq -r ' + if length == 0 then "" + else + "## Prior PR discussion (most recent up to 20 comments)\n\nIf you have already reviewed this PR (look for your own earlier comment), focus on what changed since then per the diff and respect any decisions the human made in replies. Do not re-flag findings the human already pushed back on.\n\n" + + (map("### @\(.user) (\(.created_at))\n\n\(.body)") | join("\n\n---\n\n")) + end + ' prior-comments.json > prior-comments.md + - name: Read review prompt id: review-prompt env: @@ -113,6 +131,10 @@ jobs: echo '' printf '%s\n' "$EXTRA_PROMPT" fi + if [ -s prior-comments.md ]; then + echo '' + cat prior-comments.md + fi echo 'EOF' } >> "$GITHUB_ENV" From 192866d5197c74ec930d6fe7bf9234fac76763f4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 4 May 2026 16:54:53 +0000 Subject: [PATCH 15/56] fix(flows): don't bubble error when continue_on_error is on the last step (#9029) * fix(flows): don't bubble error when continue_on_error is on the last step When the last step of a flow (or branch/forloop) failed with continue_on_error or skip_failures enabled, should_continue_flow resolved to false (because the flow was at its last step), and the flow was completed with success=false. This made parent flows / subflows treat the run as a failure even though the user explicitly asked to continue past errors. Detect this case and set success=true so the failure is captured in the result but not propagated. Co-Authored-By: Claude Opus 4.7 (1M context) * docs: explain why success is overridden post should_continue_flow Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- backend/windmill-worker/src/worker_flow.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 1577d108e8..c21f22045a 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -1587,6 +1587,16 @@ pub async fn update_flow_status_after_job_completion_internal( Ok(should_retry) }; + // For a regular module with continue_on_error at the last position, nothing else + // overrides `success` — `flow_jobs` is None so the loop/branchall override above + // doesn't fire, and `should_continue_flow` resolves to `!is_last_step = false`, + // letting the flow complete with success=false and bubble the error up to the + // enclosing job/subflow. Detect that case and treat the flow as successful. + let recoverable_failure_at_last_step = !success + && is_last_step + && !unrecoverable + && (skip_seq_branch_failure || skip_loop_failures || continue_on_error); + let should_continue_flow = match success { _ if stop_early => stop_early_err_msg.is_some() && flow_value.failure_module.is_some(), // if stop_early_err_msg some, we want to trigger the error handler before stopping the flow, if any _ if flow_job.is_canceled() => false, @@ -1606,6 +1616,10 @@ pub async fn update_flow_status_after_job_completion_internal( false => false, }; + if recoverable_failure_at_last_step { + success = true; + } + tracing::info!(id = %flow_job.id, root_id = %job_root, success = %success, stop_early = %stop_early, is_last_step = %is_last_step, unrecoverable = %unrecoverable, skip_seq_branch_failure = %skip_seq_branch_failure, skip_loop_failures = %skip_loop_failures, current_module_id = %current_module.map(|x| x.id.clone()).unwrap_or_default(), From 11b60e5b95dddd9a287ad350a7cde0f6d9e889d3 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 4 May 2026 17:22:06 +0000 Subject: [PATCH 16/56] ci: share review policy across Claude/Codex/Pi via review-prompt-shared.md (#9035) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: share review policy across Claude/Codex/Pi via review-prompt-shared.md All three reviewers now consume a single canonical policy document (.github/review-prompt-shared.md) covering AGENTS.md compliance, severity triage (P0/P1/P2), and a checklist for new public surfaces (auth contract, module placement, half-finished pub fns, input validation). Each tool's own prompt file shrinks to just its output-format quirks, and each workflow concatenates shared + tool-specific at runtime before invoking the model. Drops the suppressive "Prefer at most 10 findings" / "Keep the review high signal. If there is no clear issue, return no findings" wording from Codex and Pi, which was clipping P1 and P2 findings (e.g. half-finished pub fn, blocking I/O, wrong module placement). Replaces it with severity triage so both reviewers report all P0/P1 and surface P2 when the diff invites it. Also makes AGENTS.md authoritative for Codex (was CLAUDE.md, which is just @AGENTS.md in this repo) and adds an explicit "new public function" checklist that covers the missing-auth-check failure mode none of the three reviewers flagged on the test PR. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: move test-coverage assessment to shared prompt, slim per-tool prompts - Replace per-tool 'Reproduction instructions' with a single shared 'Test coverage assessment' section that asks each reviewer to evaluate automated coverage (sufficient / thin / appropriate) and describe what manual verification remains, if any. - Slim per-tool prompts to the absolute minimum: just where to read context, the comment header, severity tagging, and the Pi-only 'no preamble' constraint. Everything else lives in the shared policy. - Drop the model name from Pi's title ('Pi Review (DeepSeek V4)' → 'Pi Review') — the title's job is to let the bot find its own prior comment when re-reviewing; the model is irrelevant to the reader. The titles ('## Codex Review', '## Pi Review') stay because Codex and Pi both post as github-actions[bot], so the heading is the only discriminator the bot can use to find its own past comment in the prior-discussion context. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: scope test-coverage assessment to layers actually changed Don't ask reviewers about integration tests on a frontend-only diff or about playwright tests on a backend-only diff. The shared 'Test coverage' section now lists categories (backend / frontend / CI-docs) and tells the reviewer to skip the ones the PR does not touch — only ask about Rust integration tests when backend handlers/workers/queues were modified, only ask about frontend tests when components or state machines were touched, and explicitly call out 'no automated tests expected' for CI/docs/config diffs. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: don't ask reviewers to flag missing frontend component tests The Windmill frontend codebase doesn't generally test Svelte components — existing tests cover pure-logic utilities only (flowDiff, previousResults, copilot logic, dbtable queries, etc.). Asking reviewers to flag every new component for lacking a test would produce noise inconsistent with the established convention. Limit the frontend test-coverage check to new pure-logic utilities (files that would naturally have a sibling *.test.ts). Co-Authored-By: Claude Opus 4.7 (1M context) * ci: point local-review skill at the shared review policy Codex flagged (and Pi confirmed on its second pass) that slimming .claude/review-prompt.md to output-only broke the local-review skill contract — the skill still told Claude to read only that file for the review criteria, so /local-review would no longer apply severity triage, the public-surface checklist, or AGENTS.md compliance. Update the skill to read .github/review-prompt-shared.md as the policy source and .claude/review-prompt.md only for Claude output preferences. Also align the local output format with the severity-tag convention used by the workflow reviewers, and replace the lingering 'CLAUDE.md compliance' wording with 'AGENTS.md compliance'. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .claude/review-prompt.md | 28 ++------------ .claude/skills/local-review/SKILL.md | 14 ++++--- .github/codex/pr-review.prompt.md | 28 ++------------ .github/pi/pr-review.prompt.md | 28 ++------------ .github/review-prompt-shared.md | 56 +++++++++++++++++++++++++++ .github/workflows/codex-pr-review.yml | 3 +- .github/workflows/pi-pr-review.yml | 3 +- .github/workflows/pr-ready-review.yml | 4 +- 8 files changed, 82 insertions(+), 82 deletions(-) create mode 100644 .github/review-prompt-shared.md diff --git a/.claude/review-prompt.md b/.claude/review-prompt.md index c862f2d1bc..b3b6df0d74 100644 --- a/.claude/review-prompt.md +++ b/.claude/review-prompt.md @@ -1,26 +1,4 @@ -# Code Review Instructions +# Claude output format -Review this pull request and provide comprehensive feedback. - -## Focus Areas - -- **Code quality and best practices** — does the code follow established patterns? -- **Potential bugs or issues** — will this code work correctly in all cases? -- **Performance considerations** — are there unnecessary allocations, N+1 queries, or bottlenecks? -- **Security implications** — injection, auth bypass, data exposure? - -## CLAUDE.md Compliance - -Read all relevant CLAUDE.md files (root and in directories containing changed files). Check each rule against the changed code. Quote the exact rule when flagging a violation. - -## Review Guidelines - -- Provide detailed feedback using inline comments for specific issues -- Use top-level comments for general observations or praise -- Only flag issues introduced by this PR, not pre-existing problems -- Self-validate each finding: "Is this definitely a real issue?" If uncertain, discard it -- If the prompt includes a "Prior PR discussion" section, this PR has already been reviewed. Look for your own earlier comment, focus on what changed in the latest commits, and do not repeat findings the human already pushed back on or addressed - -## Testing Instructions - -At the end of your review, add complete instructions to reproduce the added changes through the app interface. These instructions will be given to a tester so they can verify the changes. It should be a short descriptive text (not a step-by-step or a list) on how to navigate the app (what page, what action, what input, etc.) to see the changes. +- Use inline comments at the relevant lines for specific issues. +- Use a top-level comment for the summary, severity-tagged finding list, AGENTS.md compliance check, and the test-coverage assessment. diff --git a/.claude/skills/local-review/SKILL.md b/.claude/skills/local-review/SKILL.md index 0399ad7294..58a9d473e2 100644 --- a/.claude/skills/local-review/SKILL.md +++ b/.claude/skills/local-review/SKILL.md @@ -6,11 +6,11 @@ description: Code review a pull request for bugs and CLAUDE.md compliance. MUST # Local Code Review Skill -Run the same review locally that the GitHub Claude Auto Review action runs on PRs. The shared review instructions live in `.claude/review-prompt.md` — read that file first and follow its instructions. +Run the same review locally that the GitHub Claude Auto Review action runs on PRs. The review policy lives in `.github/review-prompt-shared.md` (severity triage, public-surface checklist, `AGENTS.md` compliance, test-coverage assessment); `.claude/review-prompt.md` holds Claude-specific output preferences. Read both before reviewing. ## Execution Steps -1. **Read `.claude/review-prompt.md`** for the review criteria and focus areas +1. **Read `.github/review-prompt-shared.md`** for the review policy and `.claude/review-prompt.md` for the Claude output format 2. **Determine the PR scope**: - If an argument is provided, use it as the PR number or branch @@ -23,7 +23,7 @@ Run the same review locally that the GitHub Claude Auto Review action runs on PR 4. **Read changed files** where the diff alone is insufficient to understand context -5. **Apply the review instructions from `.claude/review-prompt.md`** +5. **Apply the review policy from `.github/review-prompt-shared.md`** (and the output format from `.claude/review-prompt.md`) 6. **Self-validate each finding**: Before reporting, ask yourself: - "Is this definitely a real issue, not a false positive?" @@ -39,19 +39,21 @@ Run the same review locally that the GitHub Claude Auto Review action runs on PR Found N issues: -1. () +1. [P0|P1|P2] -2. () +2. [P0|P1|P2] ``` +End with a Test coverage section per `.github/review-prompt-shared.md`. + If no issues are found: ``` ## Code review -No issues found. Checked for bugs and CLAUDE.md compliance. +No issues found. Checked for bugs, security, and AGENTS.md compliance. ``` ## Posting Comments (--comment flag) diff --git a/.github/codex/pr-review.prompt.md b/.github/codex/pr-review.prompt.md index 141fd52d61..fef52dba85 100644 --- a/.github/codex/pr-review.prompt.md +++ b/.github/codex/pr-review.prompt.md @@ -1,25 +1,5 @@ -You are reviewing a GitHub pull request for this repository. +# Codex output format -Review policy: -- Read `CLAUDE.md` before reviewing code. -- Only report issues you are confident are real and introduced by this pull request. -- Focus on bugs, security problems, and clear `CLAUDE.md` violations. -- Do not report style nits, speculative concerns, pre-existing issues, or problems that a normal linter/typechecker would obviously catch. -- Keep the review high signal. If there is no clear issue, return no findings. - -Repository context: -- Read `./.github/codex/pr-review-context.md` for the PR metadata and the exact diff commands to use. -- If the context file contains an "Additional reviewer instructions:" section, treat it as extra guidance from the human who triggered this review and follow it. -- If the context file contains a "Prior PR discussion" section, this PR has already received review activity. Look for your own previous "## Codex Review" comment, take it into account, focus on what changed in the latest commits, and do not repeat findings the human already pushed back on or addressed. -- Review only the changes introduced by this PR. -- Read additional files only when the diff is not enough to validate a finding. -- Do not modify any files. - -Output requirements: -- Return a GitHub PR comment in markdown, not JSON. -- Start with `## Codex Review`. -- Give a short overall summary first. -- If you found high-signal issues, list them in a short numbered list with file paths and line numbers when you know them confidently. -- If you found no high-signal issues, say that explicitly. -- End with a `### Reproduction instructions` section containing a short descriptive paragraph for a tester explaining how to navigate the app to observe the change. Do not make it a numbered list. If the diff is not enough to infer this safely, say that plainly. -- Prefer at most 10 findings. +- Read `./.github/codex/pr-review-context.md` for PR metadata and the diff commands. +- Return a markdown PR comment starting with `## Codex Review`. +- Tag each finding with a severity (P0 / P1 / P2), file path, and line number when known confidently. diff --git a/.github/pi/pr-review.prompt.md b/.github/pi/pr-review.prompt.md index 93ad730ba0..92f128c6b1 100644 --- a/.github/pi/pr-review.prompt.md +++ b/.github/pi/pr-review.prompt.md @@ -1,26 +1,6 @@ -You are reviewing a GitHub pull request for this repository. +# Pi output format -Review policy: -- Read `AGENTS.md` (and any `AGENTS.md` in directories containing changed files) before reviewing — it is the project's contributor guide. -- Only report issues you are confident are real and introduced by this pull request. -- Focus on bugs, security problems, and clear `AGENTS.md` violations. -- Do not report style nits, speculative concerns, pre-existing issues, or anything a normal linter/typechecker would obviously catch. -- Keep the review high signal. If there is no clear issue, return no findings. - -Repository context: -- Read `./.github/pi/pr-review-context.md` for PR metadata and the exact diff commands to use. -- If the context file contains an "Additional reviewer instructions:" section, treat it as extra guidance from the human who triggered this review and follow it. -- If the context file contains a "Prior PR discussion" section, this PR has already received review activity. Look for your own previous "## Pi Review (DeepSeek V4)" comment, take it into account, focus on what changed in the latest commits, and do not repeat findings the human already pushed back on or addressed. -- Run those `git diff` / `git log` commands to inspect the changes — they reference the base and head SHAs of this PR. -- Review only the changes introduced by this PR. Read additional files only when the diff is not enough to validate a finding. -- Do not modify any files. Do not create new files outside this review. - -Output requirements: -- Return a GitHub PR comment in markdown, not JSON. -- Start the comment with `## Pi Review (DeepSeek V4)`. -- Give a short overall summary first. -- If you found high-signal issues, list them in a short numbered list with file paths and line numbers when you know them confidently. -- If you found no high-signal issues, say that explicitly. -- End with a `### Reproduction instructions` section containing a short descriptive paragraph for a tester explaining how to navigate the app to observe the change. Do not make it a numbered list. If the diff is not enough to infer this safely, say that plainly. -- Prefer at most 10 findings. +- Read `./.github/pi/pr-review-context.md` for PR metadata and the diff commands. +- Return a markdown PR comment starting with `## Pi Review`. +- Tag each finding with a severity (P0 / P1 / P2), file path, and line number when known confidently. - Output ONLY the final review markdown — no preamble, no thinking, no tool transcripts. diff --git a/.github/review-prompt-shared.md b/.github/review-prompt-shared.md new file mode 100644 index 0000000000..2070247fd0 --- /dev/null +++ b/.github/review-prompt-shared.md @@ -0,0 +1,56 @@ +# Pull request review — shared policy + +You are reviewing a GitHub pull request for this repository. Apply this policy alongside your tool's output requirements. + +## Read the project rules first + +- Read `AGENTS.md` (repo root) and any `AGENTS.md` in directories touched by the diff before reviewing — they are the canonical contributor guide. +- `CLAUDE.md` in this repo is a wrapper around `AGENTS.md` (`@AGENTS.md`) — the same content. +- Quote the exact rule from `AGENTS.md` when flagging a violation. + +## Review policy + +- Only report issues you are confident are real and introduced by this pull request. +- Focus on bugs, security problems, performance, and clear `AGENTS.md` violations. +- Do not report style nits, speculative concerns, pre-existing issues, or anything a normal linter / typechecker would obviously catch. +- Self-validate each finding before posting: "is this definitely a real issue?" If uncertain, discard it. +- Read additional files only when the diff is not enough to validate a finding. +- Do not modify any files. + +## Severity triage + +Tag each finding with a severity. Always report P0 and P1. Report P2 only when the diff invites it (a new `pub fn`, a new module, a new exported component, a meaningful refactor). + +- **P0** — RCE, auth bypass, data loss, secrets in code, SQL injection, path traversal, broken auth on a public surface. +- **P1** — significant bug, missing auth/authorization check on a new public surface, blocking I/O on a likely async path, race condition, missing input validation on caller-controlled parameters, observable performance regression. +- **P2** — wrong module placement, doc/code mismatch, half-finished public abstractions (`pub fn` + `#[allow(dead_code)]` + `TODO`), `AGENTS.md` style violations, naming that contradicts the function's behavior. + +## Checklist for new public surfaces + +For any new `pub fn` / `pub async fn` / exported Svelte component / exported prop introduced by this PR, verify: + +- (a) auth/authorization expectations are documented in the doc comment OR enforced in the function body. A new `pub fn` that touches workspace data, secrets, files, or processes without an auth check or documented "caller MUST verify" contract is a P1. +- (b) the function is placed in a module whose stated purpose matches what it does. Check the module-level doc comment (`//!`) — a config-file reader inside `external_ip.rs` is a P2. +- (c) it is not half-finished. `pub fn` + `#[allow(dead_code)]` + a `TODO` is a smell that says the function should land together with its caller, not separately. Cite the relevant `AGENTS.md` rule. +- (d) input validation defends against injection / traversal / overflow / NUL bytes at every parameter that may be caller-controlled. + +## Test coverage assessment + +End your review with a short "Test coverage" section calibrated to the layers actually changed by the diff. Skip categories the PR does not touch. + +- **Backend** (Rust under `backend/`) — expect Rust unit tests for new logic. For new or modified API handlers, worker steps, queue/cron behavior, or DB access, also expect or note the absence of integration tests. Pure-refactor backend PRs don't need new tests if existing tests cover the surface. +- **Frontend** (Svelte / TS under `frontend/`) — the codebase does not generally test Svelte components, so do not ask for component tests. Only flag missing tests for new pure-logic utilities (the kind of file that already has a sibling `*.test.ts`, e.g. `flowDiff`, `previousResults`, copilot logic). +- **CI / workflows / docs / config-only** — no automated tests expected; say so explicitly so the reader knows you considered it. + +Then state what manual verification, if any, is still needed before merge: + +- Describe each manual scenario as a short paragraph (not a numbered list): what page / action / input, and what observable outcome confirms correctness. +- If the diff has no in-app surface to exercise (purely backend internals, CI, docs, or refactor), say that plainly. + +## Additional reviewer instructions + +If the prompt or context includes an "Additional reviewer instructions" section, treat it as extra guidance from the human who triggered this review and follow it. + +## Prior PR discussion + +If the prompt or context includes a "Prior PR discussion" section, this PR has already received review activity. Look for your own previous comment, take it into account, focus on what changed in the latest commits, and do not repeat findings the human already pushed back on or addressed. diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index 79b77f842d..fdd1d5c19a 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -263,13 +263,14 @@ jobs: - name: Run Codex review if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' run: | + cat .github/review-prompt-shared.md .github/codex/pr-review.prompt.md > /tmp/codex-prompt.md codex exec \ -C "$GITHUB_WORKSPACE" \ -m gpt-5.5 \ -c 'model_reasoning_effort="xhigh"' \ -s danger-full-access \ -o codex-final-message.md \ - - < .github/codex/pr-review.prompt.md + - < /tmp/codex-prompt.md - name: Post Codex review comment if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' diff --git a/.github/workflows/pi-pr-review.yml b/.github/workflows/pi-pr-review.yml index d7c743fc16..69767661c0 100644 --- a/.github/workflows/pi-pr-review.yml +++ b/.github/workflows/pi-pr-review.yml @@ -251,12 +251,13 @@ jobs: PI_SKIP_VERSION_CHECK: '1' run: | set -o pipefail + cat .github/review-prompt-shared.md .github/pi/pr-review.prompt.md > /tmp/pi-prompt.md pi -p \ --provider deepseek \ --model deepseek-v4-pro \ --tools read,grep,find,ls,bash \ --mode json \ - < .github/pi/pr-review.prompt.md \ + < /tmp/pi-prompt.md \ | tee pi-events.jsonl \ | jq -rc --unbuffered ' if .type == "agent_start" then "🤖 pi agent started" diff --git a/.github/workflows/pr-ready-review.yml b/.github/workflows/pr-ready-review.yml index 0058ffa8d9..342d7c37b3 100644 --- a/.github/workflows/pr-ready-review.yml +++ b/.github/workflows/pr-ready-review.yml @@ -124,10 +124,12 @@ jobs: run: | { echo 'REVIEW_PROMPT< Date: Mon, 4 May 2026 17:41:54 +0000 Subject: [PATCH 17/56] ci: cross-agent local-review skill (Claude + Pi share one file, Codex via wrapper) (#9037) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: make local-review a single-source-of-truth cross-agent skill The repo already had parallel skills directories (.agents/skills/ and .claude/skills/) drifting between agents. Consolidate local-review onto one canonical file in .agents/ and symlink the .claude/ entry to it so Claude Code and Pi share the exact same SKILL.md (Anthropic's Skills format is supported by both, only the discovery directory differs). The canonical SKILL.md now points reviewers at .github/review-prompt-shared.md as the policy source — same shared prompt the GitHub auto-review workflows already use — so local reviews and CI reviews stay in lockstep. Codex CLI doesn't support repo-level slash commands (its prompts live in ~/.codex/prompts/). For Codex parity, ship scripts/local-review.sh which pipes the SKILL + shared policy into 'codex exec' (or 'pi -p' as a uniform entry point). Update AGENTS.md to document the three invocation paths. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: make all skills cross-agent — single source in .agents/, symlink .claude/ Turn every skill into a single canonical file under .agents/skills/ and a symlink under .claude/skills/. Editing any one SKILL.md now updates all three CLIs (Claude Code reads .claude/, Codex and Pi auto-discover .agents/). Per-skill resolution: - local-review: already symlinked (prior PR #9037) - rust-backend, svelte-frontend: identical content → symlink, no edit - refine: only differed in user_invocable frontmatter → add to canonical - native-trigger: .claude/ had a newer Step 17 (sidebar visibility) missing from .agents/ → use Claude content as canonical - commit: .claude/ embedded a Claude-specific Co-Authored-By trailer the harness already injects automatically → drop from canonical, use agent-neutral .agents/ version - pr: generalize "Run /local-review" to "Invoke the local-review skill (/local-review in Claude Code, $local-review in Codex, pi --skill local-review in Pi)" and drop the Claude-specific "Generated with Claude Code" attribution from the PR body template — the harness that invoked the skill can add its own trailer if desired - adding-a-trigger: was only in .claude/ → move to .agents/ canonical - update-sqlx: was only in .agents/ → add .claude/ symlink Also drop scripts/local-review.sh — wrapper is redundant now that all three CLIs natively discover the skill from their respective directories. Update AGENTS.md accordingly. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .agents/skills/adding-a-trigger/SKILL.md | 267 ++++++++ .agents/skills/commit/SKILL.md | 1 + .agents/skills/local-review/SKILL.md | 81 +-- .agents/skills/native-trigger/SKILL.md | 13 +- .agents/skills/pr/SKILL.md | 21 +- .agents/skills/refine/SKILL.md | 1 + .claude/skills/adding-a-trigger/SKILL.md | 268 +------- .claude/skills/commit/SKILL.md | 61 +- .claude/skills/local-review/SKILL.md | 72 +- .claude/skills/native-trigger/SKILL.md | 794 +---------------------- .claude/skills/pr/SKILL.md | 112 +--- .claude/skills/refine/SKILL.md | 40 +- .claude/skills/rust-backend/SKILL.md | 108 +-- .claude/skills/svelte-frontend/SKILL.md | 81 +-- .claude/skills/update-sqlx/SKILL.md | 1 + AGENTS.md | 2 +- 16 files changed, 324 insertions(+), 1599 deletions(-) create mode 100644 .agents/skills/adding-a-trigger/SKILL.md mode change 100644 => 120000 .claude/skills/adding-a-trigger/SKILL.md mode change 100644 => 120000 .claude/skills/commit/SKILL.md mode change 100644 => 120000 .claude/skills/local-review/SKILL.md mode change 100644 => 120000 .claude/skills/native-trigger/SKILL.md mode change 100644 => 120000 .claude/skills/pr/SKILL.md mode change 100644 => 120000 .claude/skills/refine/SKILL.md mode change 100644 => 120000 .claude/skills/rust-backend/SKILL.md mode change 100644 => 120000 .claude/skills/svelte-frontend/SKILL.md create mode 120000 .claude/skills/update-sqlx/SKILL.md diff --git a/.agents/skills/adding-a-trigger/SKILL.md b/.agents/skills/adding-a-trigger/SKILL.md new file mode 100644 index 0000000000..7d8643b862 --- /dev/null +++ b/.agents/skills/adding-a-trigger/SKILL.md @@ -0,0 +1,267 @@ +--- +name: adding-a-trigger +description: Checklist for adding a new TriggerCrud-based trigger type to Windmill (Azure, GCP, Kafka, etc.). Use when wiring a new trigger kind across backend, frontend, CLI, and capture infrastructure. +--- + +# Skill: Adding a New Trigger Type + +Use this skill when adding a trigger kind that implements `TriggerCrud` (Kafka, GCP, Azure, MQTT, SQS, NATS, Postgres, Email…). For native triggers (Nextcloud, Google Drive — things wired through `windmill-native-triggers`), use the `native-trigger` skill instead. + +The goal of this doc is to enumerate every file that needs to change. Missing any one of them leads to silent regressions: sync drops the trigger, capture button does nothing, workspace forks lose it, sidebar counters undercount. Follow the checklist top-to-bottom — each section is independent enough to be validated on its own. + +Throughout this doc, substitute `{kind}` for the new trigger kind (`azure`, `kafka`, …), `{Kind}` for PascalCase (`Azure`, `Kafka`), `{KIND}` for SCREAMING (`AZURE`, `KAFKA`). + +## Reference implementations + +- **GCP** — closest analogue to Azure. Has push + pull, OIDC auth, ARM-like resource paths, capture handler. Grep for `gcp_trigger` / `GcpTrigger`. +- **Kafka** — simpler (pull-only, streaming). Good for trivial integrations. +- **Azure** — most recently added (2026). Shared-secret push auth, Event Grid namespaces + basic topics, ARM resource discovery, Namespace-pull data-plane. Grep for `azure_trigger` / `AzureTrigger`. + +## 1. Database migration + +Create a migration: `cargo sqlx migrate add -r add_{kind}_trigger` from `backend/`. Never write timestamps manually. + +The `up.sql` usually defines: +- An optional enum type (e.g. `AZURE_MODE`) if the trigger has sub-kinds +- The `{kind}_trigger` table with at minimum these columns (mirrored from kafka/gcp): + - primary: `(workspace_id, path)` + - `script_path`, `is_flow`, `enabled`, `mode`, `permissioned_as`, `edited_by`, `email` + - `edited_at`, `error`, `server_id`, `last_server_ping` + - `error_handler_path`, `error_handler_args jsonb`, `retry jsonb` + - trigger-specific fields +- Indexes on foreign keys + any frequently-filtered columns +- Foreign key to `workspace` + +Down migration drops the table and any enum types. + +## 2. Backend crate (`windmill-trigger-{kind}`) + +Create a new crate under `backend/windmill-trigger-{kind}/` with: + +- `Cargo.toml`: features `enterprise`, `private` if EE, standard deps +- `src/lib.rs`: `pub use mod_ee::*;` behind `#[cfg(all(feature = "enterprise", feature = "private"))]` +- `src/mod_ee.rs`: core types + helpers +- `src/handler_ee.rs`: `TriggerCrud` impl + route handlers +- `src/listener_ee.rs`: (only if streaming/pull-based) `Listener` trait impl + +Required in `mod_ee.rs`: +- `{Kind}Config` struct (persisted shape, `FromRow`) +- `{Kind}ConfigRequest` struct (what API receives — usually similar to Config but with validation fields) +- `{Kind}Trigger` unit struct (implements the traits) +- `impl TriggerJobArgs for {Kind}Trigger` — sets `TRIGGER_KIND`, `Payload`, `v1_payload_fn` + +Required in `handler_ee.rs`: +- `#[async_trait] impl TriggerCrud for {Kind}Trigger` with: + - `type Trigger = Trigger<{Kind}Config>` + - `type TriggerConfigRequest = {Kind}ConfigRequest` + - `const ROUTE_PREFIX: &'static str = "/{kind}_triggers";` + - `const TABLE_NAME`, `ADDITIONAL_SELECT_FIELDS` + - `get_deployed_object`, `validate_config`, `create_trigger`, `update_trigger`, `delete_trigger`, `test_connection` + - `additional_routes` (optional — mount extra endpoints for things like ARM resource listing, topic discovery) + +Register the crate in `backend/Cargo.toml` as a workspace member and as a dep of `windmill-api` behind the feature flag. + +## 3. Wire into `windmill-api` (feature-gated everywhere) + +**`backend/windmill-api/src/triggers/handler.rs`** — mount the trigger crate: +```rust +#[cfg(all(feature = "enterprise", feature = "{kind}_trigger", feature = "private"))] +{ + use crate::triggers::{kind}::{Kind}Trigger; + router = router.nest({Kind}Trigger::ROUTE_PREFIX, complete_trigger_routes({Kind}Trigger)); +} +``` + +**`backend/windmill-api/src/triggers/{kind}/mod.rs`** — re-export the crate: +```rust +pub use windmill_trigger_{kind}::*; +``` + +**`backend/windmill-api/src/lib.rs`** — if the trigger receives inbound pushes, add a webhook route: +```rust +.nest("/{kind}/w/{workspace_id}", { + #[cfg(all(feature = "enterprise", feature = "{kind}_trigger", feature = "private"))] + { triggers::{kind}::handler_oss::{kind}_push_route_handler() } + #[cfg(not(...))] + { Router::new() } +}) +``` + +## 4. `TriggerKind` enum (`backend/windmill-types/src/triggers.rs`) + +Already has slots for most triggers but verify your variant exists: +- Add `{Kind}` to the `TriggerKind` enum +- Add match arm in `to_key()` +- Add match arm in `from_str` +- Add match arm in `JobTriggerKind` (if jobs need kind tagging) + +## 5. OpenAPI (`backend/windmill-api/openapi.yaml`) + +This file is huge and the single most-forgotten place. Add: + +- `/w/{workspace}/{kind}_triggers/create` + `/update/{path}` + `/delete/{path}` + `/get/{path}` + `/list` + `/exists/{path}` + `/setmode/{path}` + `/test` paths (mirror gcp section) +- Any `additional_routes` your handler exposes (resource discovery, etc.) +- Schemas: `{Kind}Trigger`, `{Kind}TriggerData`, `{Kind}Mode` (if enum), `{Kind}DeliveryConfig`, helper request/response types +- Add `{kind}` to `CaptureTriggerKind` enum +- Add `{kind}_used: boolean` to the `UsedTriggers` response schema + +Regenerate frontend client: `npm run generate-backend-client` from `frontend/`. + +## 6. `UsedTriggers` + workspace export + +**`backend/windmill-api-workspaces/src/workspaces.rs`** — add `{kind}_used: bool` to the `UsedTriggers` struct and add an `EXISTS(SELECT 1 FROM {kind}_trigger …)` to the `get_used_triggers` query. + +**`backend/windmill-api/src/workspaces_export.rs`** — add export block mirroring gcp's (export lists all triggers, serializes them to YAML/JSON). The block re-uses the `trigger_ignore_keys` variable so the new kind automatically participates in fork-export stripping (`mode` field is omitted when the source workspace is a fork — keeps fork→parent merges from flipping the parent's enabled state). + +**Fork cloning (`clone_triggers_and_schedules` in workspaces.rs)** — add an `INSERT INTO {kind}_trigger ... SELECT ...` block that copies all rows from the parent workspace, forcing `mode = 'disabled'::TRIGGER_MODE`. Always runs at fork creation; forgetting this means users can't carry `{kind}` triggers into their forks. + +## 6.5 Hardcoded trigger-kind arrays (silent-failure hotspots) + +Several files keep **hardcoded arrays** of trigger kind strings. Miss one and ACL checks / user offboarding / trash drop your kind: + +- **`backend/windmill-api-groups/src/granular_acls.rs`** — `KINDS: [&str; N]`. **Increment N** (the compile error is cryptic otherwise). Controls which kinds accept granular ACL operations. +- **`backend/windmill-api-users/src/users.rs`** (`extra_perms_tables`) — which tables get `extra_perms` entries cleaned when a user is deleted. +- **`backend/windmill-api/src/offboarding.rs`** — three separate arrays (enumeration, fork-copy, and delete paths). **All three** need the new kind. +- **`backend/windmill-api/src/trash.rs`** — `valid_tables` for the trash / restore API. +- **`backend/windmill-git-sync/src/lib.rs`** — add a test assertion for `DeployedObject::{Kind}Trigger.get_kind() == "{kind}_trigger"` (the `get_kind` match arm itself lives in the enum impl — already required by the Rust compiler). +- **`backend/windmill-api-auth/src/scopes.rs`** — add the `{Kind}Triggers` variant to `ScopeDomain` enum + `as_str` match + `from_str` match. Required for the OAuth/token system to recognise `{kind}_triggers:read|write` scopes. +- **`backend/windmill-api/src/token.rs`** (`build_trigger_scope_domains` → `TRIGGER_DOMAINS`) — add `("{kind}_triggers", "{Kind display name}")` so the CreateToken UI's scope selector surfaces the `read` / `write` checkboxes. + +**OpenAPI enums** to extend (do NOT forget — generated client will allow it but server rejects as 400): +- `CaptureTriggerKind` enum +- Three `kind` enums under `/w/{workspace}/acls/{get,add,remove}/{kind}/{path}` (yes, same list repeated three times) + +After editing any of these, run a full `cargo check` with your feature flag + `gcp_trigger` + other core flags — the `KINDS: [&str; N]` length mismatch only surfaces when the crate compiles. + +## 7. Capture infrastructure (`backend/windmill-api/src/capture.rs`) + +If the trigger supports push delivery, it also needs a capture endpoint so users can test it: + +- `{Kind}TriggerConfig` struct (gated by feature flags) +- `TriggerConfig::{Kind}` variant +- `set_{kind}_trigger_config` function (creates the subscription/equivalent pointing at the capture URL — use your `manage_{kind}_subscription` helper with `trigger_mode=false`) +- Both real + no-op versions behind feature gates +- `TriggerKind::{Kind} => set_{kind}_trigger_config(...)` arm in `set_config` +- `{kind}_payload` async handler — validates auth (if any), processes payload, calls `insert_capture_payload` +- Route: `.route("/{kind}/{runnable_kind}/{*path}", post({kind}_payload))` inside `workspaced_unauthed_service` — and expand the surrounding `#[cfg(any(...))]` to include your feature flag + +## 8. CLI (`cli/`) — easy to miss, breaks sync silently + +Check all of these: + +**`cli/src/types.ts`:** +- Add `"{kind}"` to `TRIGGER_TYPES` array +- Add `"{kind}_trigger"` to `getTypeStrFromPath` return union +- Add match case in `getTypeStrFromPath`'s `typeEnding ===` chain +- Add `pushTrigger("{kind}", ...)` branch in `pushObj` + +**`cli/src/commands/trigger/trigger.ts`:** +- Import `{Kind}Trigger` type +- Add `{kind}: {Kind}Trigger` to the `Trigger` type map +- Add `{kind}: wmill.get{Kind}Trigger`, `update{Kind}Trigger`, `create{Kind}Trigger` to each function map +- Add `{kind}: { ... }` template to `triggerTemplates` +- Add `list{Kind}Triggers` call + spread in the `list` aggregation +- Update `--kind` option descriptions to mention the new kind + +**`cli/src/commands/sync/sync.ts`:** +- Add `path.endsWith(".{kind}_trigger" + ext)` in the file-type filter +- Add `typ == "{kind}_trigger"` in `getTypeOrder` +- Add `"{kind}_trigger"` to the delete-suffix regex (~line 3092) +- Add a `case "{kind}_trigger"` in the delete switch + +**`cli/src/guidance/skills.ts`** — **DO NOT EDIT DIRECTLY**. It's auto-generated by `system_prompts/generate.py`. Instead: +- Edit `system_prompts/utils.py` → append `('{Kind}Trigger', '{kind}_trigger')` to the `SCHEMA_MAPPINGS['triggers']` list (this is the master list — the one in `generate.py` is duplicated and `utils.py` wins) +- Then run `python3 system_prompts/generate.py` — it regenerates `cli/src/guidance/skills.ts` with the schema extracted from `backend/windmill-api/openapi.yaml` +- Commit the regenerated file + +## 9. Frontend — editor + drawer + +Under `frontend/src/lib/components/triggers/{kind}/`: + +- `{Kind}TriggerPanel.svelte` — the tile shown in the triggers listing +- `{Kind}TriggerEditor.svelte` — outer drawer wrapper +- `{Kind}TriggerEditorInner.svelte` — state + business logic; must expose: + - `openEdit(path, isFlow, defaultValues?)` method + - `isEditor` prop, `onConfigChange` + `onCaptureConfigChange` callbacks + - `get{Kind}Config()` + `get{Kind}CaptureConfig()` helpers + - `captureConfig = $derived.by(untrack(() => isEditor) ? get{Kind}CaptureConfig : () => ({}))` + - `$effect(() => { const args = [captureConfig, isValid] as const; untrack(() => onCaptureConfigChange?.(...args)) })` +- `{Kind}TriggerEditorConfigSection.svelte` — form fields; use design-system components (`TextInput`, `Select`, `Toggle`, `ToggleButtonGroup`), never raw `` +- `{Kind}Capture.svelte` — capture panel; wraps `CaptureSection` with `captureType="{kind}"` +- `utils.ts` — `requestBody` builders and any trigger-type-specific helpers + +## 10. Frontend — global integration + +Easy to miss: + +- **`frontend/src/lib/components/triggers.ts`** — add `'{kind}'` to the `TriggerKind` union +- **`frontend/src/lib/components/triggers/CaptureWrapper.svelte`**: + - Import `{Kind}Capture` + - Add to `isStreamingCapture()` array (streaming = pull-style; push-style is typically `false`) + - Add `{:else if captureType === '{kind}'}` branch with the `<{Kind}Capture>` render +- **`frontend/src/lib/components/sidebar/SidebarContent.svelte`** — import the icon, add the nav entry +- **`frontend/src/lib/components/sidebar/OperatorMenu.svelte`** — add the operator-mode entry +- **`frontend/src/routes/(root)/(logged)/+layout.svelte`** — destructure `{kind}_used` from `/get_used_triggers` response, push `'{kind}'` into `usedKinds` +- **`frontend/src/lib/components/search/GlobalSearchModal.svelte`** — import icon, add "Go to {Kind} ..." entry +- **`frontend/src/lib/components/offboarding-utils.ts`** — add mappings `{kind}_trigger: '{kind}_triggers'` and `{kind}_trigger: '{kind} trigger'` +- **`frontend/src/lib/components/icons/{Kind}Icon.svelte`** — single-path SVG, `fill={color ?? 'currentColor'}`, `size` prop default 16 (match existing icons — don't hardcode colors, don't use `width`/`height` props) +- **`frontend/src/routes/(root)/(logged)/{kind}_triggers/+page.svelte`** — listing page (mirror `gcp_triggers/+page.svelte` for push+pull, `kafka_triggers` for pure streaming) +- **`frontend/src/lib/components/CompareWorkspaces.svelte`** — workspace fork / compare tool. Needs: service import, editor import, `{kind}Editor` `$state`, `case '{kind}'` in `openTriggerDetails()`, entry in `triggerServices` object (list/delete/normalize), and `<{Kind}TriggerEditor bind:this={{kind}Editor} />` in the template + +## 10.5 AI system prompts (`system_prompts/`) + +- **`system_prompts/utils.py`** — append `('{Kind}Trigger', '{kind}_trigger')` to `SCHEMA_MAPPINGS['triggers']` (master list used by code generation + CLI skills) +- **`system_prompts/generate.py`** — also has a duplicated `schema_types` list (~line 903) for the AI `triggers` skill content. Add `('{Kind}Trigger', '{kind}_trigger')` there too +- **`system_prompts/generate.py`** `schema_names` (~line 1192) — add `'{Kind}Trigger'` (add `'New{Kind}Trigger'` only if the OpenAPI declares one; GCP and Azure don't) +- Run `python3 system_prompts/generate.py` — this rewrites `cli/src/guidance/skills.ts` and all `auto-generated/` docs. Commit the regenerated files + +## 11. Validation + +Run all of these before declaring done: + +```bash +# Backend +cd backend +cargo check --features enterprise,{kind}_trigger,private # minimal +cargo check --features enterprise,azure_trigger,private,gcp_trigger,http_trigger,mqtt_trigger,postgres_trigger,sqs_trigger,kafka,nats,smtp,websocket # full + +# SQLx offline data (never run `cargo sqlx prepare` directly — use the wrapper) +./update_sqlx.sh + +# Frontend +cd frontend +npm run generate-backend-client +npm run check:fast +``` + +Smoke test in the UI: create a trigger, save, check it appears in sidebar + search, delete, re-create via CLI `wmill sync`. + +## 12. Common pitfalls + +- **Forgetting feature gates in `workspaced_unauthed_service()`** — the surrounding `#[cfg(any(...))]` expression must include your feature flag, not just the inner `#[cfg]` on the route +- **`.route(path, ...).route(path, ...)` with same path and different methods** — older axum replaced; use `.route(path, post(h1).options(h2))` to chain methods on the same `MethodRouter` +- **`on:event` directives** — legacy Svelte 4, no-op in runes mode. Use callback props (`onSelected`, `onConfigChange`) +- **`$bindable(default_value)` on optional props** — banned by project CLAUDE.md. Use `$bindable()` + `$derived(prop ?? default)` instead +- **CORS layer intercepting OPTIONS** — tower-http CorsLayer short-circuits OPTIONS before reaching your handler. For server-to-server webhook endpoints, drop the CORS layer entirely (CORS is browser-only) +- **DeliveryAttributeMappings / custom headers for auth** — prefer HMAC or sha256-hashed shared secrets over opaque JWTs when the provider doesn't support signed tokens natively. Store only the hash; regenerate secret on every save +- **ARM / API resource-listing cascades** — if the trigger's resource type is deep (Azure: subscription → RG → namespace → topic), offer dropdowns in the UI populated from the provider's APIs using the user's credential resource +- **Clearing stale selections on dependency change** — when a dropdown's underlying data reloads (e.g., user changes SP or edition), clear selections that no longer match the new list +- **Workspace-scoped tag compatibility** — if the trigger has tags, verify forked workspaces handle them (see commit `0773b5bc85` for a historical fix) + +## 13. EE file split + +If the trigger is enterprise-only, the code lives in `windmill-ee-private__worktrees/.../windmill-trigger-{kind}/src/*_ee.rs` and is symlinked into the OSS tree. The `windmill-ee-private__worktrees/` directory holds the real files; changes propagate via symlinks. See `docs/enterprise.md` for the workflow. + +## 14. Final checklist before PR + +- [ ] Migration up/down tested (revert + re-apply) +- [ ] `./update_sqlx.sh` committed the updated `.sqlx/` offline data +- [ ] `cargo check` passes with your feature flag + with all trigger features +- [ ] `npm run check:fast` passes +- [ ] Trigger visible in sidebar with correct icon weight (not oversized/colored — use `currentColor`) +- [ ] Create, edit, delete flow all work in the UI +- [ ] Capture button works (if push-capable) +- [ ] Trigger appears in `/get_used_triggers` → sidebar pulse +- [ ] `wmill sync pull` + `wmill sync push` both round-trip the trigger +- [ ] `wmill trigger list` includes it +- [ ] OpenAPI schemas are complete (no `null` in generated types) diff --git a/.agents/skills/commit/SKILL.md b/.agents/skills/commit/SKILL.md index 3f97552466..114531570e 100644 --- a/.agents/skills/commit/SKILL.md +++ b/.agents/skills/commit/SKILL.md @@ -1,5 +1,6 @@ --- name: commit +user_invocable: true description: Create a git commit with conventional commit format. MUST use anytime you want to commit changes. --- diff --git a/.agents/skills/local-review/SKILL.md b/.agents/skills/local-review/SKILL.md index ad701ac367..3477911176 100644 --- a/.agents/skills/local-review/SKILL.md +++ b/.agents/skills/local-review/SKILL.md @@ -1,97 +1,66 @@ --- name: local-review -description: Code review a pull request for bugs and CLAUDE.md compliance. MUST use when asked to review code. +description: Code review the current PR (or branch diff against main) for bugs, security, and AGENTS.md compliance. MUST use when asked to review code. --- -# Local Code Review Skill +# Local Code Review -Review a pull request for real bugs and CLAUDE.md compliance violations. This review targets HIGH SIGNAL issues only. +Run the same review locally that the GitHub auto-review actions run on PRs (Claude / Codex / Pi). The review policy lives in `.github/review-prompt-shared.md` — read that first. -## Review Philosophy +## Steps -- **Only flag issues you are certain about.** If you are not sure an issue is real, do not flag it. False positives erode trust and waste reviewer time. -- Think like a senior engineer doing a final review — flag things that would cause incidents, not things that are merely imperfect. +1. **Read `.github/review-prompt-shared.md`** for the review policy: severity triage (P0 / P1 / P2), the new-public-surface checklist, AGENTS.md compliance, and the test-coverage assessment. -## What to Flag - -- Code that won't compile or parse (syntax errors, type errors, missing imports) -- Code that will definitely produce wrong results regardless of inputs -- Clear, unambiguous CLAUDE.md violations (quote the exact rule being violated) -- Security issues in introduced code (injection, auth bypass, data exposure) -- Incorrect logic that will fail in production - -## What NOT to Flag - -- Code style or quality concerns -- Potential issues that depend on specific inputs or runtime state -- Subjective suggestions or improvements -- Pre-existing issues not introduced by this PR -- Pedantic nitpicks a senior engineer wouldn't flag -- Issues a linter or type checker will catch -- General quality concerns unless explicitly prohibited in CLAUDE.md -- Issues silenced via lint ignore comments - -## Execution Steps - -1. **Determine the PR scope**: - - If an argument is provided, use it as the PR number or branch - - Otherwise, detect from the current branch vs main - - Run `gh pr view` if a PR exists, or use `git diff main...HEAD` - -2. **Find relevant CLAUDE.md files**: - - Read the root `CLAUDE.md` - - Check for CLAUDE.md files in directories containing changed files +2. **Determine the PR scope**: + - If an argument is provided, treat it as a PR number or branch. + - Otherwise, detect from the current branch vs `main`. + - Run `gh pr view` if a PR exists; otherwise compare against `main` with `git diff main...HEAD`. 3. **Get the diff and metadata**: - - `gh pr diff` or `git diff main...HEAD` for the full diff - - `gh pr view` or `git log main..HEAD --oneline` for context + - `gh pr diff` or `git diff main...HEAD` for the full diff. + - `gh pr view` or `git log main..HEAD --oneline` for context. -4. **Read changed files** where the diff alone is insufficient to understand context +4. **Read changed files** when the diff alone is insufficient. -5. **Review for**: - - CLAUDE.md compliance — check each rule against the changed code - - Bugs and logic errors — will this code work correctly? - - Security issues — injection, auth, data exposure in new code +5. **Apply the policy** from `.github/review-prompt-shared.md`. Self-validate each finding before reporting (real issue? would a senior engineer flag it?). -6. **Self-validate each finding**: Before reporting, ask yourself: - - "Is this definitely a real issue, not a false positive?" - - "Would a senior engineer flag this in review?" - - If the answer to either is no, discard the finding +6. **Output findings** to the terminal (default) or post as PR comments (with `--comment` flag). -7. **Output findings** to the terminal (default) or post as PR comments (with `--comment` flag) - -## Output Format +## Output format ``` ## Code review Found N issues: -1. () +1. [P0|P1|P2] -2. () +2. [P0|P1|P2] ``` +End with a `Test coverage` section per the shared policy. + If no issues are found: ``` ## Code review -No issues found. Checked for bugs and CLAUDE.md compliance. +No issues found. Checked for bugs, security, and AGENTS.md compliance. ``` -## Posting Comments (--comment flag) +## Posting comments (`--comment`) -If the user passes `--comment`, post findings as inline PR comments using: +For a top-level PR comment: ```bash gh pr review --comment --body "" ``` -Or for inline comments on specific lines: +For inline comments on specific lines: ```bash -gh api repos/{owner}/{repo}/pulls/{pr}/reviews -f body="" -f event="COMMENT" -f comments="[...]" +gh api repos/{owner}/{repo}/pulls/{pr}/reviews \ + -f body="" -f event="COMMENT" -f comments="[...]" ``` diff --git a/.agents/skills/native-trigger/SKILL.md b/.agents/skills/native-trigger/SKILL.md index 781e200d0c..ae38f70b32 100644 --- a/.agents/skills/native-trigger/SKILL.md +++ b/.agents/skills/native-trigger/SKILL.md @@ -607,7 +607,18 @@ In `frontend/src/lib/components/triggers/TriggersEditor.svelte`: Add your service to the `nativeTriggerServices` map in `deleteDeployedTrigger()`. Native triggers use `NativeTriggerService.deleteNativeTrigger({ workspace, serviceName, externalId })` instead of the standard `path`-based delete. -### Step 17: Update OpenAPI Spec and Regenerate Types +### Step 17: Update `getUsedTriggers` for Sidebar Visibility + +The sidebar (`frontend/src/lib/components/sidebar/SidebarContent.svelte`) shows native-trigger links only if `$usedTriggerKinds` includes the service — without this, your trigger page will never appear in the nav bar even when triggers exist. + +1. **Backend** — add `{service}_used: bool` to the `UsedTriggers` struct and SELECT in `backend/windmill-api-workspaces/src/workspaces.rs::get_used_triggers()`: + ```rust + EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = '{service}'::native_trigger_service) AS "{service}_used!" + ``` +2. **OpenAPI** — add `{service}_used: boolean` to the response schema for `GET /w/{workspace}/workspaces/used_triggers` (under both `properties` and `required`). +3. **Layout** — in `frontend/src/routes/(root)/(logged)/+layout.svelte::loadUsedTriggerKinds()`, destructure `{service}_used` and push `'{service}'` to `usedKinds`. + +### Step 18: Update OpenAPI Spec and Regenerate Types Add to `JobTriggerKind` enum in `backend/windmill-api/openapi.yaml`, then: diff --git a/.agents/skills/pr/SKILL.md b/.agents/skills/pr/SKILL.md index 2efcc4e0a6..ef52d6e110 100644 --- a/.agents/skills/pr/SKILL.md +++ b/.agents/skills/pr/SKILL.md @@ -1,5 +1,6 @@ --- name: pr +user_invocable: true description: Open a draft pull request on GitHub. MUST use when you want to create/open a PR. --- @@ -50,22 +51,22 @@ The body MUST be explicit about what changed. Structure: ## Test plan - [ ] - [ ] - ---- -Generated with [Claude Code](https://claude.com/claude-code) ``` +The harness/tooling that invoked the skill may add its own attribution trailer; the skill itself does not prescribe one. + ## Execution Steps 1. Run `git status` to check for uncommitted changes 2. Run `git log main..HEAD --oneline` to see all commits in this branch 3. Run `git diff main...HEAD` to see the full diff against main -4. Check if remote branch exists and is up to date: +4. **Invoke the `local-review` skill** before creating the PR (`/local-review` in Claude Code, `$local-review` in Codex, `pi --skill local-review` / `/skill:local-review` in Pi). If issues are found, fix them and commit before proceeding. Do not skip this step. +5. Check if remote branch exists and is up to date: ```bash git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "no upstream" ``` -5. Push to remote if needed: `git push -u origin HEAD` -6. Create draft PR using gh CLI: +6. Push to remote if needed: `git push -u origin HEAD` +7. Create draft PR using gh CLI: ```bash gh pr create --draft --title ": " --body "$(cat <<'EOF' ## Summary @@ -78,13 +79,10 @@ Generated with [Claude Code](https://claude.com/claude-code) ## Test plan - [ ] - [ ] - - --- - Generated with [Claude Code](https://claude.com/claude-code) EOF )" ``` -7. Return the PR URL to the user +8. Return the PR URL to the user ## EE Companion PR (when `*_ee.rs` files were modified) @@ -100,9 +98,6 @@ Follow the full EE PR workflow in `docs/enterprise.md`. The key PR-specific deta ```bash gh pr create --draft --repo windmill-labs/windmill-ee-private --title ": " --body "$(cat <<'EOF' Companion PR for windmill-labs/windmill# - - --- - Generated with [Claude Code](https://claude.com/claude-code) EOF )" ``` diff --git a/.agents/skills/refine/SKILL.md b/.agents/skills/refine/SKILL.md index b96e97e8a2..aaf747cd29 100644 --- a/.agents/skills/refine/SKILL.md +++ b/.agents/skills/refine/SKILL.md @@ -1,5 +1,6 @@ --- name: refine +user_invocable: true description: End-of-session reflection. Reviews friction encountered during the session and proposes updates to docs/ to capture lessons learned. --- diff --git a/.claude/skills/adding-a-trigger/SKILL.md b/.claude/skills/adding-a-trigger/SKILL.md deleted file mode 100644 index 7d8643b862..0000000000 --- a/.claude/skills/adding-a-trigger/SKILL.md +++ /dev/null @@ -1,267 +0,0 @@ ---- -name: adding-a-trigger -description: Checklist for adding a new TriggerCrud-based trigger type to Windmill (Azure, GCP, Kafka, etc.). Use when wiring a new trigger kind across backend, frontend, CLI, and capture infrastructure. ---- - -# Skill: Adding a New Trigger Type - -Use this skill when adding a trigger kind that implements `TriggerCrud` (Kafka, GCP, Azure, MQTT, SQS, NATS, Postgres, Email…). For native triggers (Nextcloud, Google Drive — things wired through `windmill-native-triggers`), use the `native-trigger` skill instead. - -The goal of this doc is to enumerate every file that needs to change. Missing any one of them leads to silent regressions: sync drops the trigger, capture button does nothing, workspace forks lose it, sidebar counters undercount. Follow the checklist top-to-bottom — each section is independent enough to be validated on its own. - -Throughout this doc, substitute `{kind}` for the new trigger kind (`azure`, `kafka`, …), `{Kind}` for PascalCase (`Azure`, `Kafka`), `{KIND}` for SCREAMING (`AZURE`, `KAFKA`). - -## Reference implementations - -- **GCP** — closest analogue to Azure. Has push + pull, OIDC auth, ARM-like resource paths, capture handler. Grep for `gcp_trigger` / `GcpTrigger`. -- **Kafka** — simpler (pull-only, streaming). Good for trivial integrations. -- **Azure** — most recently added (2026). Shared-secret push auth, Event Grid namespaces + basic topics, ARM resource discovery, Namespace-pull data-plane. Grep for `azure_trigger` / `AzureTrigger`. - -## 1. Database migration - -Create a migration: `cargo sqlx migrate add -r add_{kind}_trigger` from `backend/`. Never write timestamps manually. - -The `up.sql` usually defines: -- An optional enum type (e.g. `AZURE_MODE`) if the trigger has sub-kinds -- The `{kind}_trigger` table with at minimum these columns (mirrored from kafka/gcp): - - primary: `(workspace_id, path)` - - `script_path`, `is_flow`, `enabled`, `mode`, `permissioned_as`, `edited_by`, `email` - - `edited_at`, `error`, `server_id`, `last_server_ping` - - `error_handler_path`, `error_handler_args jsonb`, `retry jsonb` - - trigger-specific fields -- Indexes on foreign keys + any frequently-filtered columns -- Foreign key to `workspace` - -Down migration drops the table and any enum types. - -## 2. Backend crate (`windmill-trigger-{kind}`) - -Create a new crate under `backend/windmill-trigger-{kind}/` with: - -- `Cargo.toml`: features `enterprise`, `private` if EE, standard deps -- `src/lib.rs`: `pub use mod_ee::*;` behind `#[cfg(all(feature = "enterprise", feature = "private"))]` -- `src/mod_ee.rs`: core types + helpers -- `src/handler_ee.rs`: `TriggerCrud` impl + route handlers -- `src/listener_ee.rs`: (only if streaming/pull-based) `Listener` trait impl - -Required in `mod_ee.rs`: -- `{Kind}Config` struct (persisted shape, `FromRow`) -- `{Kind}ConfigRequest` struct (what API receives — usually similar to Config but with validation fields) -- `{Kind}Trigger` unit struct (implements the traits) -- `impl TriggerJobArgs for {Kind}Trigger` — sets `TRIGGER_KIND`, `Payload`, `v1_payload_fn` - -Required in `handler_ee.rs`: -- `#[async_trait] impl TriggerCrud for {Kind}Trigger` with: - - `type Trigger = Trigger<{Kind}Config>` - - `type TriggerConfigRequest = {Kind}ConfigRequest` - - `const ROUTE_PREFIX: &'static str = "/{kind}_triggers";` - - `const TABLE_NAME`, `ADDITIONAL_SELECT_FIELDS` - - `get_deployed_object`, `validate_config`, `create_trigger`, `update_trigger`, `delete_trigger`, `test_connection` - - `additional_routes` (optional — mount extra endpoints for things like ARM resource listing, topic discovery) - -Register the crate in `backend/Cargo.toml` as a workspace member and as a dep of `windmill-api` behind the feature flag. - -## 3. Wire into `windmill-api` (feature-gated everywhere) - -**`backend/windmill-api/src/triggers/handler.rs`** — mount the trigger crate: -```rust -#[cfg(all(feature = "enterprise", feature = "{kind}_trigger", feature = "private"))] -{ - use crate::triggers::{kind}::{Kind}Trigger; - router = router.nest({Kind}Trigger::ROUTE_PREFIX, complete_trigger_routes({Kind}Trigger)); -} -``` - -**`backend/windmill-api/src/triggers/{kind}/mod.rs`** — re-export the crate: -```rust -pub use windmill_trigger_{kind}::*; -``` - -**`backend/windmill-api/src/lib.rs`** — if the trigger receives inbound pushes, add a webhook route: -```rust -.nest("/{kind}/w/{workspace_id}", { - #[cfg(all(feature = "enterprise", feature = "{kind}_trigger", feature = "private"))] - { triggers::{kind}::handler_oss::{kind}_push_route_handler() } - #[cfg(not(...))] - { Router::new() } -}) -``` - -## 4. `TriggerKind` enum (`backend/windmill-types/src/triggers.rs`) - -Already has slots for most triggers but verify your variant exists: -- Add `{Kind}` to the `TriggerKind` enum -- Add match arm in `to_key()` -- Add match arm in `from_str` -- Add match arm in `JobTriggerKind` (if jobs need kind tagging) - -## 5. OpenAPI (`backend/windmill-api/openapi.yaml`) - -This file is huge and the single most-forgotten place. Add: - -- `/w/{workspace}/{kind}_triggers/create` + `/update/{path}` + `/delete/{path}` + `/get/{path}` + `/list` + `/exists/{path}` + `/setmode/{path}` + `/test` paths (mirror gcp section) -- Any `additional_routes` your handler exposes (resource discovery, etc.) -- Schemas: `{Kind}Trigger`, `{Kind}TriggerData`, `{Kind}Mode` (if enum), `{Kind}DeliveryConfig`, helper request/response types -- Add `{kind}` to `CaptureTriggerKind` enum -- Add `{kind}_used: boolean` to the `UsedTriggers` response schema - -Regenerate frontend client: `npm run generate-backend-client` from `frontend/`. - -## 6. `UsedTriggers` + workspace export - -**`backend/windmill-api-workspaces/src/workspaces.rs`** — add `{kind}_used: bool` to the `UsedTriggers` struct and add an `EXISTS(SELECT 1 FROM {kind}_trigger …)` to the `get_used_triggers` query. - -**`backend/windmill-api/src/workspaces_export.rs`** — add export block mirroring gcp's (export lists all triggers, serializes them to YAML/JSON). The block re-uses the `trigger_ignore_keys` variable so the new kind automatically participates in fork-export stripping (`mode` field is omitted when the source workspace is a fork — keeps fork→parent merges from flipping the parent's enabled state). - -**Fork cloning (`clone_triggers_and_schedules` in workspaces.rs)** — add an `INSERT INTO {kind}_trigger ... SELECT ...` block that copies all rows from the parent workspace, forcing `mode = 'disabled'::TRIGGER_MODE`. Always runs at fork creation; forgetting this means users can't carry `{kind}` triggers into their forks. - -## 6.5 Hardcoded trigger-kind arrays (silent-failure hotspots) - -Several files keep **hardcoded arrays** of trigger kind strings. Miss one and ACL checks / user offboarding / trash drop your kind: - -- **`backend/windmill-api-groups/src/granular_acls.rs`** — `KINDS: [&str; N]`. **Increment N** (the compile error is cryptic otherwise). Controls which kinds accept granular ACL operations. -- **`backend/windmill-api-users/src/users.rs`** (`extra_perms_tables`) — which tables get `extra_perms` entries cleaned when a user is deleted. -- **`backend/windmill-api/src/offboarding.rs`** — three separate arrays (enumeration, fork-copy, and delete paths). **All three** need the new kind. -- **`backend/windmill-api/src/trash.rs`** — `valid_tables` for the trash / restore API. -- **`backend/windmill-git-sync/src/lib.rs`** — add a test assertion for `DeployedObject::{Kind}Trigger.get_kind() == "{kind}_trigger"` (the `get_kind` match arm itself lives in the enum impl — already required by the Rust compiler). -- **`backend/windmill-api-auth/src/scopes.rs`** — add the `{Kind}Triggers` variant to `ScopeDomain` enum + `as_str` match + `from_str` match. Required for the OAuth/token system to recognise `{kind}_triggers:read|write` scopes. -- **`backend/windmill-api/src/token.rs`** (`build_trigger_scope_domains` → `TRIGGER_DOMAINS`) — add `("{kind}_triggers", "{Kind display name}")` so the CreateToken UI's scope selector surfaces the `read` / `write` checkboxes. - -**OpenAPI enums** to extend (do NOT forget — generated client will allow it but server rejects as 400): -- `CaptureTriggerKind` enum -- Three `kind` enums under `/w/{workspace}/acls/{get,add,remove}/{kind}/{path}` (yes, same list repeated three times) - -After editing any of these, run a full `cargo check` with your feature flag + `gcp_trigger` + other core flags — the `KINDS: [&str; N]` length mismatch only surfaces when the crate compiles. - -## 7. Capture infrastructure (`backend/windmill-api/src/capture.rs`) - -If the trigger supports push delivery, it also needs a capture endpoint so users can test it: - -- `{Kind}TriggerConfig` struct (gated by feature flags) -- `TriggerConfig::{Kind}` variant -- `set_{kind}_trigger_config` function (creates the subscription/equivalent pointing at the capture URL — use your `manage_{kind}_subscription` helper with `trigger_mode=false`) -- Both real + no-op versions behind feature gates -- `TriggerKind::{Kind} => set_{kind}_trigger_config(...)` arm in `set_config` -- `{kind}_payload` async handler — validates auth (if any), processes payload, calls `insert_capture_payload` -- Route: `.route("/{kind}/{runnable_kind}/{*path}", post({kind}_payload))` inside `workspaced_unauthed_service` — and expand the surrounding `#[cfg(any(...))]` to include your feature flag - -## 8. CLI (`cli/`) — easy to miss, breaks sync silently - -Check all of these: - -**`cli/src/types.ts`:** -- Add `"{kind}"` to `TRIGGER_TYPES` array -- Add `"{kind}_trigger"` to `getTypeStrFromPath` return union -- Add match case in `getTypeStrFromPath`'s `typeEnding ===` chain -- Add `pushTrigger("{kind}", ...)` branch in `pushObj` - -**`cli/src/commands/trigger/trigger.ts`:** -- Import `{Kind}Trigger` type -- Add `{kind}: {Kind}Trigger` to the `Trigger` type map -- Add `{kind}: wmill.get{Kind}Trigger`, `update{Kind}Trigger`, `create{Kind}Trigger` to each function map -- Add `{kind}: { ... }` template to `triggerTemplates` -- Add `list{Kind}Triggers` call + spread in the `list` aggregation -- Update `--kind` option descriptions to mention the new kind - -**`cli/src/commands/sync/sync.ts`:** -- Add `path.endsWith(".{kind}_trigger" + ext)` in the file-type filter -- Add `typ == "{kind}_trigger"` in `getTypeOrder` -- Add `"{kind}_trigger"` to the delete-suffix regex (~line 3092) -- Add a `case "{kind}_trigger"` in the delete switch - -**`cli/src/guidance/skills.ts`** — **DO NOT EDIT DIRECTLY**. It's auto-generated by `system_prompts/generate.py`. Instead: -- Edit `system_prompts/utils.py` → append `('{Kind}Trigger', '{kind}_trigger')` to the `SCHEMA_MAPPINGS['triggers']` list (this is the master list — the one in `generate.py` is duplicated and `utils.py` wins) -- Then run `python3 system_prompts/generate.py` — it regenerates `cli/src/guidance/skills.ts` with the schema extracted from `backend/windmill-api/openapi.yaml` -- Commit the regenerated file - -## 9. Frontend — editor + drawer - -Under `frontend/src/lib/components/triggers/{kind}/`: - -- `{Kind}TriggerPanel.svelte` — the tile shown in the triggers listing -- `{Kind}TriggerEditor.svelte` — outer drawer wrapper -- `{Kind}TriggerEditorInner.svelte` — state + business logic; must expose: - - `openEdit(path, isFlow, defaultValues?)` method - - `isEditor` prop, `onConfigChange` + `onCaptureConfigChange` callbacks - - `get{Kind}Config()` + `get{Kind}CaptureConfig()` helpers - - `captureConfig = $derived.by(untrack(() => isEditor) ? get{Kind}CaptureConfig : () => ({}))` - - `$effect(() => { const args = [captureConfig, isValid] as const; untrack(() => onCaptureConfigChange?.(...args)) })` -- `{Kind}TriggerEditorConfigSection.svelte` — form fields; use design-system components (`TextInput`, `Select`, `Toggle`, `ToggleButtonGroup`), never raw `` -- `{Kind}Capture.svelte` — capture panel; wraps `CaptureSection` with `captureType="{kind}"` -- `utils.ts` — `requestBody` builders and any trigger-type-specific helpers - -## 10. Frontend — global integration - -Easy to miss: - -- **`frontend/src/lib/components/triggers.ts`** — add `'{kind}'` to the `TriggerKind` union -- **`frontend/src/lib/components/triggers/CaptureWrapper.svelte`**: - - Import `{Kind}Capture` - - Add to `isStreamingCapture()` array (streaming = pull-style; push-style is typically `false`) - - Add `{:else if captureType === '{kind}'}` branch with the `<{Kind}Capture>` render -- **`frontend/src/lib/components/sidebar/SidebarContent.svelte`** — import the icon, add the nav entry -- **`frontend/src/lib/components/sidebar/OperatorMenu.svelte`** — add the operator-mode entry -- **`frontend/src/routes/(root)/(logged)/+layout.svelte`** — destructure `{kind}_used` from `/get_used_triggers` response, push `'{kind}'` into `usedKinds` -- **`frontend/src/lib/components/search/GlobalSearchModal.svelte`** — import icon, add "Go to {Kind} ..." entry -- **`frontend/src/lib/components/offboarding-utils.ts`** — add mappings `{kind}_trigger: '{kind}_triggers'` and `{kind}_trigger: '{kind} trigger'` -- **`frontend/src/lib/components/icons/{Kind}Icon.svelte`** — single-path SVG, `fill={color ?? 'currentColor'}`, `size` prop default 16 (match existing icons — don't hardcode colors, don't use `width`/`height` props) -- **`frontend/src/routes/(root)/(logged)/{kind}_triggers/+page.svelte`** — listing page (mirror `gcp_triggers/+page.svelte` for push+pull, `kafka_triggers` for pure streaming) -- **`frontend/src/lib/components/CompareWorkspaces.svelte`** — workspace fork / compare tool. Needs: service import, editor import, `{kind}Editor` `$state`, `case '{kind}'` in `openTriggerDetails()`, entry in `triggerServices` object (list/delete/normalize), and `<{Kind}TriggerEditor bind:this={{kind}Editor} />` in the template - -## 10.5 AI system prompts (`system_prompts/`) - -- **`system_prompts/utils.py`** — append `('{Kind}Trigger', '{kind}_trigger')` to `SCHEMA_MAPPINGS['triggers']` (master list used by code generation + CLI skills) -- **`system_prompts/generate.py`** — also has a duplicated `schema_types` list (~line 903) for the AI `triggers` skill content. Add `('{Kind}Trigger', '{kind}_trigger')` there too -- **`system_prompts/generate.py`** `schema_names` (~line 1192) — add `'{Kind}Trigger'` (add `'New{Kind}Trigger'` only if the OpenAPI declares one; GCP and Azure don't) -- Run `python3 system_prompts/generate.py` — this rewrites `cli/src/guidance/skills.ts` and all `auto-generated/` docs. Commit the regenerated files - -## 11. Validation - -Run all of these before declaring done: - -```bash -# Backend -cd backend -cargo check --features enterprise,{kind}_trigger,private # minimal -cargo check --features enterprise,azure_trigger,private,gcp_trigger,http_trigger,mqtt_trigger,postgres_trigger,sqs_trigger,kafka,nats,smtp,websocket # full - -# SQLx offline data (never run `cargo sqlx prepare` directly — use the wrapper) -./update_sqlx.sh - -# Frontend -cd frontend -npm run generate-backend-client -npm run check:fast -``` - -Smoke test in the UI: create a trigger, save, check it appears in sidebar + search, delete, re-create via CLI `wmill sync`. - -## 12. Common pitfalls - -- **Forgetting feature gates in `workspaced_unauthed_service()`** — the surrounding `#[cfg(any(...))]` expression must include your feature flag, not just the inner `#[cfg]` on the route -- **`.route(path, ...).route(path, ...)` with same path and different methods** — older axum replaced; use `.route(path, post(h1).options(h2))` to chain methods on the same `MethodRouter` -- **`on:event` directives** — legacy Svelte 4, no-op in runes mode. Use callback props (`onSelected`, `onConfigChange`) -- **`$bindable(default_value)` on optional props** — banned by project CLAUDE.md. Use `$bindable()` + `$derived(prop ?? default)` instead -- **CORS layer intercepting OPTIONS** — tower-http CorsLayer short-circuits OPTIONS before reaching your handler. For server-to-server webhook endpoints, drop the CORS layer entirely (CORS is browser-only) -- **DeliveryAttributeMappings / custom headers for auth** — prefer HMAC or sha256-hashed shared secrets over opaque JWTs when the provider doesn't support signed tokens natively. Store only the hash; regenerate secret on every save -- **ARM / API resource-listing cascades** — if the trigger's resource type is deep (Azure: subscription → RG → namespace → topic), offer dropdowns in the UI populated from the provider's APIs using the user's credential resource -- **Clearing stale selections on dependency change** — when a dropdown's underlying data reloads (e.g., user changes SP or edition), clear selections that no longer match the new list -- **Workspace-scoped tag compatibility** — if the trigger has tags, verify forked workspaces handle them (see commit `0773b5bc85` for a historical fix) - -## 13. EE file split - -If the trigger is enterprise-only, the code lives in `windmill-ee-private__worktrees/.../windmill-trigger-{kind}/src/*_ee.rs` and is symlinked into the OSS tree. The `windmill-ee-private__worktrees/` directory holds the real files; changes propagate via symlinks. See `docs/enterprise.md` for the workflow. - -## 14. Final checklist before PR - -- [ ] Migration up/down tested (revert + re-apply) -- [ ] `./update_sqlx.sh` committed the updated `.sqlx/` offline data -- [ ] `cargo check` passes with your feature flag + with all trigger features -- [ ] `npm run check:fast` passes -- [ ] Trigger visible in sidebar with correct icon weight (not oversized/colored — use `currentColor`) -- [ ] Create, edit, delete flow all work in the UI -- [ ] Capture button works (if push-capable) -- [ ] Trigger appears in `/get_used_triggers` → sidebar pulse -- [ ] `wmill sync pull` + `wmill sync push` both round-trip the trigger -- [ ] `wmill trigger list` includes it -- [ ] OpenAPI schemas are complete (no `null` in generated types) diff --git a/.claude/skills/adding-a-trigger/SKILL.md b/.claude/skills/adding-a-trigger/SKILL.md new file mode 120000 index 0000000000..a2060ad897 --- /dev/null +++ b/.claude/skills/adding-a-trigger/SKILL.md @@ -0,0 +1 @@ +../../../.agents/skills/adding-a-trigger/SKILL.md \ No newline at end of file diff --git a/.claude/skills/commit/SKILL.md b/.claude/skills/commit/SKILL.md deleted file mode 100644 index 2094dbab06..0000000000 --- a/.claude/skills/commit/SKILL.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -name: commit -user_invocable: true -description: Create a git commit with conventional commit format. MUST use anytime you want to commit changes. ---- - -# Git Commit Skill - -Create a focused, single-line commit following conventional commit conventions. - -## Instructions - -1. **Analyze changes**: Run `git status` and `git diff` to understand what was modified -2. **Stage only modified files**: Add files individually by name. NEVER use `git add -A` or `git add .` -3. **Write commit message**: Follow the conventional commit format as a single line - -## Conventional Commit Format - -``` -: -``` - -### Types -- `feat`: New feature or capability -- `fix`: Bug fix -- `refactor`: Code change that neither fixes a bug nor adds a feature -- `docs`: Documentation only changes -- `style`: Formatting, missing semicolons, etc (no code change) -- `test`: Adding or correcting tests -- `chore`: Maintenance tasks, dependency updates, etc -- `perf`: Performance improvement - -### Rules -- Message MUST be a single line (no multi-line messages) -- Description should be lowercase, imperative mood ("add" not "added") -- No period at the end -- Keep under 72 characters total - -### Examples -``` -feat: add token usage tracking for AI providers -fix: resolve null pointer in job executor -refactor: extract common validation logic -docs: update API endpoint documentation -chore: upgrade sqlx to 0.7 -``` - -## Execution Steps - -1. Run `git status` to see all changes -2. Run `git diff` to understand the changes in detail -3. Run `git log --oneline -5` to see recent commit style -4. Stage ONLY the modified/relevant files: `git add ...` -5. Create the commit with conventional format: - ```bash - git commit -m ": - - Co-Authored-By: Claude Opus 4.5 " - ``` -6. Run `git status` to verify the commit succeeded diff --git a/.claude/skills/commit/SKILL.md b/.claude/skills/commit/SKILL.md new file mode 120000 index 0000000000..11493a3d1e --- /dev/null +++ b/.claude/skills/commit/SKILL.md @@ -0,0 +1 @@ +../../../.agents/skills/commit/SKILL.md \ No newline at end of file diff --git a/.claude/skills/local-review/SKILL.md b/.claude/skills/local-review/SKILL.md deleted file mode 100644 index 58a9d473e2..0000000000 --- a/.claude/skills/local-review/SKILL.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -name: local-review -user_invocable: true -description: Code review a pull request for bugs and CLAUDE.md compliance. MUST use when asked to review code. ---- - -# Local Code Review Skill - -Run the same review locally that the GitHub Claude Auto Review action runs on PRs. The review policy lives in `.github/review-prompt-shared.md` (severity triage, public-surface checklist, `AGENTS.md` compliance, test-coverage assessment); `.claude/review-prompt.md` holds Claude-specific output preferences. Read both before reviewing. - -## Execution Steps - -1. **Read `.github/review-prompt-shared.md`** for the review policy and `.claude/review-prompt.md` for the Claude output format - -2. **Determine the PR scope**: - - If an argument is provided, use it as the PR number or branch - - Otherwise, detect from the current branch vs main - - Run `gh pr view` if a PR exists, or use `git diff main...HEAD` - -3. **Get the diff and metadata**: - - `gh pr diff` or `git diff main...HEAD` for the full diff - - `gh pr view` or `git log main..HEAD --oneline` for context - -4. **Read changed files** where the diff alone is insufficient to understand context - -5. **Apply the review policy from `.github/review-prompt-shared.md`** (and the output format from `.claude/review-prompt.md`) - -6. **Self-validate each finding**: Before reporting, ask yourself: - - "Is this definitely a real issue, not a false positive?" - - "Would a senior engineer flag this in review?" - - If the answer to either is no, discard the finding - -7. **Output findings** to the terminal (default) or post as PR comments (with `--comment` flag) - -## Output Format - -``` -## Code review - -Found N issues: - -1. [P0|P1|P2] - - -2. [P0|P1|P2] - -``` - -End with a Test coverage section per `.github/review-prompt-shared.md`. - -If no issues are found: - -``` -## Code review - -No issues found. Checked for bugs, security, and AGENTS.md compliance. -``` - -## Posting Comments (--comment flag) - -If the user passes `--comment`, post findings as inline PR comments using: - -```bash -gh pr review --comment --body "" -``` - -Or for inline comments on specific lines: - -```bash -gh api repos/{owner}/{repo}/pulls/{pr}/reviews -f body="" -f event="COMMENT" -f comments="[...]" -``` diff --git a/.claude/skills/local-review/SKILL.md b/.claude/skills/local-review/SKILL.md new file mode 120000 index 0000000000..8072aff10d --- /dev/null +++ b/.claude/skills/local-review/SKILL.md @@ -0,0 +1 @@ +../../../.agents/skills/local-review/SKILL.md \ No newline at end of file diff --git a/.claude/skills/native-trigger/SKILL.md b/.claude/skills/native-trigger/SKILL.md deleted file mode 100644 index ae38f70b32..0000000000 --- a/.claude/skills/native-trigger/SKILL.md +++ /dev/null @@ -1,793 +0,0 @@ ---- -name: native-trigger -description: Guidance for adding native trigger services to Windmill. Use when implementing or modifying native trigger integrations across the backend and frontend. ---- - -# Skill: Adding Native Trigger Services - -This skill provides comprehensive guidance for adding new native trigger services to Windmill. Native triggers allow external services (like Nextcloud, Google Drive, etc.) to trigger Windmill scripts/flows via webhooks or push notifications. - -## Architecture Overview - -The native trigger system consists of: - -1. **Database Layer** - PostgreSQL tables and enum types -2. **Backend Rust Implementation** - Core trait, handlers, and service modules in the `windmill-native-triggers` crate -3. **Frontend Svelte Components** - Configuration forms and UI components - -### Key Files - -| Component | Path | -|-----------|------| -| Core module with `External` trait | `backend/windmill-native-triggers/src/lib.rs` | -| Generic CRUD handlers | `backend/windmill-native-triggers/src/handler.rs` | -| Background sync logic | `backend/windmill-native-triggers/src/sync.rs` | -| OAuth/workspace integration | `backend/windmill-native-triggers/src/workspace_integrations.rs` | -| Re-export shim (windmill-api) | `backend/windmill-api/src/native_triggers/mod.rs` | -| TriggerKind enum | `backend/windmill-common/src/triggers.rs` | -| JobTriggerKind enum | `backend/windmill-common/src/jobs.rs` | -| Frontend service registry | `frontend/src/lib/components/triggers/native/utils.ts` | -| Frontend trigger utilities | `frontend/src/lib/components/triggers/utils.ts` | -| Trigger badges (icons + counts) | `frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte` | -| Workspace integrations UI | `frontend/src/lib/components/workspaceSettings/WorkspaceIntegrations.svelte` | -| OAuth config form component | `frontend/src/lib/components/workspaceSettings/OAuthClientConfig.svelte` | -| OpenAPI spec | `backend/windmill-api/openapi.yaml` | -| Reference: Nextcloud module | `backend/windmill-native-triggers/src/nextcloud/` | -| Reference: Google module | `backend/windmill-native-triggers/src/google/` | - -### Crate Structure - -The native trigger code lives in the `windmill-native-triggers` crate (`backend/windmill-native-triggers/`). The `windmill-api` crate re-exports everything via a shim: - -```rust -// backend/windmill-api/src/native_triggers/mod.rs -pub use windmill_native_triggers::*; -``` - -All new service modules go in `backend/windmill-native-triggers/src/`. - ---- - -## Core Concepts - -### The `External` Trait - -Every native trigger service implements the `External` trait defined in `lib.rs`: - -```rust -#[async_trait] -pub trait External: Send + Sync + 'static { - // Associated types: - type ServiceConfig: Debug + DeserializeOwned + Serialize + Send + Sync; - type TriggerData: Debug + Serialize + Send + Sync; - type OAuthData: DeserializeOwned + Serialize + Clone + Send + Sync; - type CreateResponse: DeserializeOwned + Send + Sync; - - // Constants: - const SUPPORT_WEBHOOK: bool; - const SERVICE_NAME: ServiceName; - const DISPLAY_NAME: &'static str; - const TOKEN_ENDPOINT: &'static str; - const REFRESH_ENDPOINT: &'static str; - const AUTH_ENDPOINT: &'static str; - - // Required methods: - async fn create(&self, w_id, oauth_data, webhook_token, data, db, tx) -> Result; - async fn update(&self, w_id, oauth_data, external_id, webhook_token, data, db, tx) -> Result; - async fn get(&self, w_id, oauth_data, external_id, db, tx) -> Result; - async fn delete(&self, w_id, oauth_data, external_id, db, tx) -> Result<()>; - async fn exists(&self, w_id, oauth_data, external_id, db, tx) -> Result; - async fn maintain_triggers(&self, db, workspace_id, triggers, oauth_data, synced, errors); - fn external_id_and_metadata_from_response(&self, resp) -> (String, Option); - - // Methods with defaults: - async fn prepare_webhook(&self, db, w_id, headers, body, script_path, is_flow) -> Result; - fn service_config_from_create_response(&self, data, resp) -> Option; - fn additional_routes(&self) -> axum::Router; - async fn http_client_request(&self, url, method, workspace_id, tx, db, headers, body) -> Result; -} -``` - -Key design points: -- **`update()` returns `serde_json::Value`** - the resolved service_config to store. Each service is responsible for building the final config. -- **`maintain_triggers()`** - periodic background maintenance. Each service implements its own strategy (Nextcloud: reconcile with external state; Google: renew expiring channels). -- **No `list_all()` in the trait** - services that need it (Nextcloud) implement it privately; services that don't (Google) use different maintenance strategies. -- **No `get_external_id_from_trigger_data()` or `extract_service_config_from_trigger_data()`** - removed in favor of the `maintain_triggers` pattern. - -### Create Lifecycle: Two Paths - -The `create_native_trigger` handler in `handler.rs` supports two creation flows, controlled by `service_config_from_create_response()`: - -**Path A: Short (Google pattern)** - `service_config_from_create_response()` returns `Some(config)`: -1. `create()` registers on external service -2. `external_id_and_metadata_from_response()` extracts the ID -3. `service_config_from_create_response()` builds the config directly from input data + response metadata -4. Stores trigger in DB -- done, no extra round-trip - -Use this when the external_id is known before the create call (e.g., Google generates the channel_id as a UUID upfront and includes it in the webhook URL). - -**Path B: Long (Nextcloud pattern)** - `service_config_from_create_response()` returns `None` (default): -1. `create()` registers on external service (webhook URL has no external_id yet) -2. `external_id_and_metadata_from_response()` extracts the ID -3. `update()` is called to fix the webhook URL with the now-known external_id -4. `update()` returns the resolved service_config -5. Stores trigger in DB - -Use this when the external_id is assigned by the remote service and the webhook URL needs to be corrected after creation. - -### OAuth Token Storage (Three-Table Pattern) - -OAuth tokens are stored across three tables, NOT in `workspace_integrations.oauth_data` directly: - -| Table | What's Stored | -|-------|---------------| -| `workspace_integrations` | `oauth_data` JSON with `base_url`, `client_id`, `client_secret`, `instance_shared` flag; `resource_path` pointing to the variable | -| `variable` | Encrypted `access_token` (at the path stored in `resource_path`), linked to `account` via `account` column | -| `account` | `refresh_token`, keyed by `workspace_id` + `client` (service name) + `is_workspace_integration = true` | - -The `decrypt_oauth_data()` function in `lib.rs` assembles these into a unified struct: -```rust -pub struct OAuthConfig { - pub base_url: String, - pub access_token: String, // decrypted from variable - pub refresh_token: Option, // from account table - pub client_id: String, // from oauth_data or instance settings - pub client_secret: String, // from oauth_data or instance settings -} -``` - -Instance-level sharing: when `oauth_data.instance_shared == true`, `client_id` and `client_secret` are read from global settings instead of workspace_integrations. - -### URL Resolution - -The `resolve_endpoint()` helper handles both absolute and relative OAuth URLs: - -```rust -pub fn resolve_endpoint(base_url: &str, endpoint: &str) -> String { - if endpoint.starts_with("http://") || endpoint.starts_with("https://") { - endpoint.to_string() // Google: absolute URLs - } else { - format!("{}{}", base_url, endpoint) // Nextcloud: relative paths - } -} -``` - -### ServiceName Methods - -`ServiceName` is the central registry enum. Each variant must implement these match arms: - -| Method | Purpose | -|--------|---------| -| `as_str()` | Lowercase identifier (e.g., `"google"`) | -| `as_trigger_kind()` | Maps to `TriggerKind` enum | -| `as_job_trigger_kind()` | Maps to `JobTriggerKind` enum | -| `token_endpoint()` | OAuth token endpoint (relative or absolute) | -| `auth_endpoint()` | OAuth authorization endpoint | -| `oauth_scopes()` | Space-separated OAuth scopes | -| `resource_type()` | Resource type for token storage (e.g., `"gworkspace"`) | -| `extra_auth_params()` | Extra OAuth params (e.g., Google needs `access_type=offline`, `prompt=consent`) | -| `integration_service()` | Maps to the workspace integration service (usually `*self`) | -| `TryFrom` | Parse from string | -| `Display` | Delegates to `as_str()` | - ---- - -## Step-by-Step Implementation Guide - -### Step 1: Database Migration - -Create a new migration file: `backend/migrations/YYYYMMDDHHMMSS_newservice_trigger.up.sql` - -```sql --- Add the service to the native_trigger_service enum -ALTER TYPE native_trigger_service ADD VALUE IF NOT EXISTS 'newservice'; - --- Add to TRIGGER_KIND enum (used for trigger tracking) -ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'newservice'; - --- Add to job_trigger_kind enum (used for job tracking) -ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'newservice'; -``` - -Also create the corresponding down migration. - -### Step 2: Update windmill-common Enums - -#### `backend/windmill-common/src/triggers.rs` - -Add variant to `TriggerKind` enum, and update `to_key()` and `fmt()` implementations. - -#### `backend/windmill-common/src/jobs.rs` - -Add variant to `JobTriggerKind` enum and update the `Display` implementation. - -### Step 3: Backend Service Module - -Create a new directory: `backend/windmill-native-triggers/src/newservice/` - -#### `mod.rs` - Type Definitions - -```rust -use serde::{Deserialize, Serialize}; - -pub mod external; -// pub mod routes; // Only if you need additional service-specific routes - -/// OAuth data deserialized from the three-table pattern. -/// The actual structure is built by decrypt_oauth_data() from variable + account + workspace_integrations. -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct NewServiceOAuthData { - pub base_url: String, // from workspace_integrations.oauth_data - pub access_token: String, // decrypted from variable table - pub refresh_token: Option, // from account table - // Note: client_id and client_secret are in OAuthConfig, not here - // unless the service needs them at runtime for API calls -} - -/// Configuration provided by user when creating/updating a trigger. -/// Stored as JSON in native_trigger.service_config. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct NewServiceConfig { - // Service-specific configuration fields - pub folder_path: String, - pub file_filter: Option, -} - -/// Data retrieved from the external service about a trigger. -/// Returned by the get() method and shown in the UI. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct NewServiceTriggerData { - pub folder_path: String, - pub file_filter: Option, - // Fields that shouldn't affect service_config comparison should use #[serde(skip_serializing)] -} - -/// Response from external service when creating a trigger/webhook. -#[derive(Debug, Deserialize)] -pub struct CreateTriggerResponse { - pub id: String, -} - -/// Handler struct (stateless, used for routing) -#[derive(Copy, Clone)] -pub struct NewService; -``` - -#### `external.rs` - External Trait Implementation - -```rust -use async_trait::async_trait; -use reqwest::Method; -use sqlx::PgConnection; -use std::collections::HashMap; -use windmill_common::{ - error::{Error, Result}, - BASE_URL, DB, -}; - -use crate::{ - generate_webhook_service_url, External, NativeTrigger, NativeTriggerData, ServiceName, - sync::{SyncError, TriggerSyncInfo}, -}; -use super::{NewService, NewServiceConfig, NewServiceOAuthData, NewServiceTriggerData, CreateTriggerResponse}; - -#[async_trait] -impl External for NewService { - type ServiceConfig = NewServiceConfig; - type TriggerData = NewServiceTriggerData; - type OAuthData = NewServiceOAuthData; - type CreateResponse = CreateTriggerResponse; - - const SERVICE_NAME: ServiceName = ServiceName::NewService; - const DISPLAY_NAME: &'static str = "New Service"; - const SUPPORT_WEBHOOK: bool = true; - const TOKEN_ENDPOINT: &'static str = "/oauth/token"; - const REFRESH_ENDPOINT: &'static str = "/oauth/token"; - const AUTH_ENDPOINT: &'static str = "/oauth/authorize"; - - async fn create( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - webhook_token: &str, - data: &NativeTriggerData, - db: &DB, - tx: &mut PgConnection, - ) -> Result { - let base_url = &*BASE_URL.read().await; - - // external_id is None during create (we get it from the response) - let webhook_url = generate_webhook_service_url( - base_url, w_id, &data.script_path, data.is_flow, - None, Self::SERVICE_NAME, webhook_token, - ); - - let url = format!("{}/api/webhooks/create", oauth_data.base_url); - let payload = serde_json::json!({ - "callback_url": webhook_url, - "folder_path": data.service_config.folder_path, - }); - - let response: CreateTriggerResponse = self - .http_client_request(&url, Method::POST, w_id, tx, db, None, Some(&payload)) - .await?; - - Ok(response) - } - - /// Update returns the resolved service_config as JSON. - /// For services using the update+get pattern, call self.get() and serialize. - async fn update( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - webhook_token: &str, - data: &NativeTriggerData, - db: &DB, - tx: &mut PgConnection, - ) -> Result { - let base_url = &*BASE_URL.read().await; - - let webhook_url = generate_webhook_service_url( - base_url, w_id, &data.script_path, data.is_flow, - Some(external_id), Self::SERVICE_NAME, webhook_token, - ); - - let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); - let payload = serde_json::json!({ - "callback_url": webhook_url, - "folder_path": data.service_config.folder_path, - }); - - let _: serde_json::Value = self - .http_client_request(&url, Method::PUT, w_id, tx, db, None, Some(&payload)) - .await?; - - // Fetch back the updated state to get the resolved config - let trigger_data = self.get(w_id, oauth_data, external_id, db, tx).await?; - serde_json::to_value(&trigger_data) - .map_err(|e| Error::InternalErr(format!("Failed to serialize trigger data: {}", e))) - } - - async fn get( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result { - let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); - self.http_client_request::<_, ()>(&url, Method::GET, w_id, tx, db, None, None).await - } - - async fn delete( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result<()> { - let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); - let _: serde_json::Value = self - .http_client_request::<_, ()>(&url, Method::DELETE, w_id, tx, db, None, None) - .await - .or_else(|e| match &e { - Error::InternalErr(msg) if msg.contains("404") => Ok(serde_json::Value::Null), - _ => Err(e), - })?; - Ok(()) - } - - async fn exists( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result { - match self.get(w_id, oauth_data, external_id, db, tx).await { - Ok(_) => Ok(true), - Err(Error::NotFound(_)) => Ok(false), - Err(e) => Err(e), - } - } - - /// Background maintenance. Choose the right pattern for your service: - /// - For services with queryable external state: use reconcile_with_external_state() - /// - For channel-based services with expiration: implement renewal logic - async fn maintain_triggers( - &self, - db: &DB, - workspace_id: &str, - triggers: &[NativeTrigger], - oauth_data: &Self::OAuthData, - synced: &mut Vec, - errors: &mut Vec, - ) { - // Option A: Reconcile with external state (Nextcloud pattern) - // Fetch all triggers from external service and compare with DB - let external_triggers = match self.list_all(workspace_id, oauth_data, db).await { - Ok(triggers) => triggers, - Err(e) => { - errors.push(SyncError { - resource_path: format!("workspace:{}", workspace_id), - error_message: format!("Failed to list triggers: {}", e), - error_type: "api_error".to_string(), - }); - return; - } - }; - - // Convert to (external_id, config_json) pairs - let external_pairs: Vec<(String, serde_json::Value)> = external_triggers - .into_iter() - .map(|t| (t.id.clone(), serde_json::to_value(&t).unwrap_or_default())) - .collect(); - - crate::sync::reconcile_with_external_state( - db, workspace_id, Self::SERVICE_NAME, triggers, &external_pairs, synced, errors, - ).await; - } - - fn external_id_and_metadata_from_response( - &self, - resp: &Self::CreateResponse, - ) -> (String, Option) { - (resp.id.clone(), None) - } - - // service_config_from_create_response: NOT overridden (returns None). - // This means the handler uses the update+get pattern after create. - // Override and return Some(...) to skip the update+get cycle (Google pattern). -} - -impl NewService { - /// Private helper to list all triggers from the external service. - async fn list_all( - &self, - w_id: &str, - oauth_data: &::OAuthData, - db: &DB, - ) -> Result::TriggerData>> { - // Implementation depends on the external service's API - todo!() - } -} -``` - -### Step 4: Update lib.rs Registry - -In `backend/windmill-native-triggers/src/lib.rs`: - -```rust -// Service modules - add new services here: -#[cfg(feature = "native_trigger")] -pub mod newservice; // <-- Add this - -// ServiceName enum - add variant: -pub enum ServiceName { - Nextcloud, - Google, - NewService, // <-- Add this -} - -// Then add match arms in ALL ServiceName methods: -// as_str(), as_trigger_kind(), as_job_trigger_kind(), token_endpoint(), -// auth_endpoint(), oauth_scopes(), resource_type(), extra_auth_params(), -// integration_service(), TryFrom, Display -``` - -### Step 5: Update handler.rs Routes - -In `backend/windmill-native-triggers/src/handler.rs`: - -```rust -pub fn generate_native_trigger_routers() -> Router { - // ... - #[cfg(feature = "native_trigger")] - { - use crate::newservice::NewService; - return router - .nest("/nextcloud", service_routes(NextCloud)) - .nest("/google", service_routes(Google)) - .nest("/newservice", service_routes(NewService)); // <-- Add this - } - // ... -} -``` - -### Step 6: Update sync.rs - -In `backend/windmill-native-triggers/src/sync.rs`: - -```rust -pub async fn sync_all_triggers(db: &DB) -> Result { - // ... - #[cfg(feature = "native_trigger")] - { - use crate::newservice::NewService; - - // ... existing service syncs ... - - // New service sync - let (service_name, result) = sync_service_triggers(db, NewService).await; - total_synced += result.synced_triggers.len(); - total_errors += result.errors.len(); - service_results.insert(service_name, result); - } - // ... -} -``` - -### Step 7: Frontend Service Registry - -In `frontend/src/lib/components/triggers/native/utils.ts`: - -Add to `NATIVE_TRIGGER_SERVICES`, `getTriggerIconName()`, and `getServiceIcon()`. - -### Step 8: Frontend Trigger Form Component - -Create: `frontend/src/lib/components/triggers/native/services/newservice/NewServiceTriggerForm.svelte` - -### Step 9: Frontend Icon Component - -Create: `frontend/src/lib/components/icons/NewServiceIcon.svelte` - -### Step 10: Update NativeTriggerEditor - -Check `frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte` to ensure it dynamically loads form components based on service name. - -### Step 11: Workspace Integration UI - -Add your service to the `supportedServices` map in `frontend/src/lib/components/workspaceSettings/WorkspaceIntegrations.svelte`: - -```typescript -const supportedServices: Record = { - // ... existing services ... - newservice: { - name: 'newservice', - displayName: 'New Service', - description: 'Connect to New Service for triggers', - icon: NewServiceIcon, - docsUrl: 'https://www.windmill.dev/docs/integrations/newservice', - requiresBaseUrl: false, // false for cloud services, true for self-hosted - setupInstructions: [ - 'Step 1: Create an OAuth app on the service', - 'Step 2: Configure the redirect URI shown below', - 'Step 3: Enter the client credentials below' - ] - } -} -``` - -### Step 12: Update `frontend/src/lib/components/triggers/utils.ts` - -Update ALL of these maps/functions: -1. `triggerIconMap` - import and add icon -2. `triggerDisplayNamesMap` - add display name -3. `triggerTypeOrder` in `sortTriggers()` - add type -4. `getLightConfig()` - add case for your service -5. `getTriggerLabel()` - add case for your service -6. `jobTriggerKinds` - add to array -7. `countPropertyMap` - add count property -8. `triggerSaveFunctions` - add save function - -### Step 13: Update TriggersBadge Component - -In `frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte`: - -1. Import the icon -2. Add to `baseConfig` with `countKey` (the dynamic `availableNativeServices` loop does NOT set `countKey`) -3. Add to the `allTypes` array - -### Step 14: Update TriggersWrapper.svelte - -In `frontend/src/lib/components/triggers/TriggersWrapper.svelte`: - -Add a `{:else if selectedTrigger.type === 'yourservice'}` case that renders `` with the same props pattern as the existing native trigger cases (e.g., `nextcloud`). - -### Step 15: Update AddTriggersButton.svelte - -In `frontend/src/lib/components/triggers/AddTriggersButton.svelte`: - -1. Add `yourserviceAvailable` state variable -2. Add `setYourserviceState()` async function using `isServiceAvailable('yourservice', $workspaceStore!)` -3. Call it at module level -4. Add a dropdown entry to `addTriggerItems` with `hidden: !yourserviceAvailable` - -### Step 16: Update TriggersEditor.svelte Delete Handling - -In `frontend/src/lib/components/triggers/TriggersEditor.svelte`: - -Add your service to the `nativeTriggerServices` map in `deleteDeployedTrigger()`. Native triggers use `NativeTriggerService.deleteNativeTrigger({ workspace, serviceName, externalId })` instead of the standard `path`-based delete. - -### Step 17: Update `getUsedTriggers` for Sidebar Visibility - -The sidebar (`frontend/src/lib/components/sidebar/SidebarContent.svelte`) shows native-trigger links only if `$usedTriggerKinds` includes the service — without this, your trigger page will never appear in the nav bar even when triggers exist. - -1. **Backend** — add `{service}_used: bool` to the `UsedTriggers` struct and SELECT in `backend/windmill-api-workspaces/src/workspaces.rs::get_used_triggers()`: - ```rust - EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = '{service}'::native_trigger_service) AS "{service}_used!" - ``` -2. **OpenAPI** — add `{service}_used: boolean` to the response schema for `GET /w/{workspace}/workspaces/used_triggers` (under both `properties` and `required`). -3. **Layout** — in `frontend/src/routes/(root)/(logged)/+layout.svelte::loadUsedTriggerKinds()`, destructure `{service}_used` and push `'{service}'` to `usedKinds`. - -### Step 18: Update OpenAPI Spec and Regenerate Types - -Add to `JobTriggerKind` enum in `backend/windmill-api/openapi.yaml`, then: - -```bash -cd frontend && npm run generate-backend-client -``` - ---- - -## Special Patterns - -### Unified Service with `trigger_type` (Google Pattern) - -When a single service handles multiple trigger types (e.g., Google Drive + Calendar share OAuth and API patterns), use a single `ServiceName` variant with a discriminator field: - -```rust -pub enum GoogleTriggerType { Drive, Calendar } - -pub struct GoogleServiceConfig { - pub trigger_type: GoogleTriggerType, - // Drive-specific fields (only used when trigger_type = Drive) - pub resource_id: Option, - pub resource_name: Option, - // Calendar-specific fields (only used when trigger_type = Calendar) - pub calendar_id: Option, - pub calendar_name: Option, - // Metadata set after creation - pub google_resource_id: Option, - pub expiration: Option, -} -``` - -Branch in trait methods based on `trigger_type`. Frontend uses a `ToggleButtonGroup` to switch between types. This keeps the codebase simpler (one service, one OAuth flow, one set of routes). - -See `backend/windmill-native-triggers/src/google/` for the reference implementation. - -### Skipping update+get After Create (Google Pattern) - -Override `service_config_from_create_response()` to return `Some(config)` when the external_id is known before the create call: - -```rust -fn service_config_from_create_response( - &self, - data: &NativeTriggerData, - resp: &Self::CreateResponse, -) -> Option { - // Clone input config, add metadata from response - let mut config = data.service_config.clone(); - config.google_resource_id = Some(resp.resource_id.clone()); - config.expiration = Some(resp.expiration.clone()); - Some(serde_json::to_value(&config).unwrap()) -} -``` - -### Services with Absolute OAuth Endpoints (Google) - -Unlike self-hosted services where OAuth endpoints are relative paths appended to `base_url`, services like Google have absolute URLs: - -```rust -// Nextcloud: relative paths -ServiceName::Nextcloud => "/apps/oauth2/api/v1/token", -// Google: absolute URLs -ServiceName::Google => "https://oauth2.googleapis.com/token", -``` - -The `resolve_endpoint()` function handles both. For services with absolute endpoints: -- `base_url` can be empty -- `requiresBaseUrl: false` in the frontend workspace integration config -- Add `extra_auth_params()` if needed (Google requires `access_type=offline` and `prompt=consent`) - -### Channel-Based Push Notifications with Renewal (Google Pattern) - -For services using expiring watch channels instead of persistent webhooks: - -1. Store expiration in `service_config` (as part of `ServiceConfig`) -2. In `maintain_triggers()`, implement renewal logic instead of using `reconcile_with_external_state()`: - ```rust - async fn maintain_triggers(&self, db, workspace_id, triggers, oauth_data, synced, errors) { - for trigger in triggers { - if should_renew_channel(trigger) { - self.renew_channel(db, trigger, oauth_data).await; - } - } - } - ``` -3. Renewal: best-effort stop old channel, create new one with same external_id, update service_config with new expiration -4. Google example: Drive channels expire in 24h (renew when <1h left), Calendar channels expire in 7 days (renew when <1 day left) - -### reconcile_with_external_state (Nextcloud Pattern) - -The reusable function in `sync.rs` compares external triggers with DB state: -- Triggers missing externally: sets error "Trigger no longer exists on external service" -- Triggers present externally: clears errors, updates service_config if it differs - -Usage in `maintain_triggers()`: -```rust -let external_pairs: Vec<(String, serde_json::Value)> = /* fetch from external */; -crate::sync::reconcile_with_external_state( - db, workspace_id, Self::SERVICE_NAME, triggers, &external_pairs, synced, errors, -).await; -``` - -### Webhook Payload Processing - -Override `prepare_webhook()` to parse service-specific payloads into script/flow args: - -```rust -async fn prepare_webhook(&self, db, w_id, headers, body, script_path, is_flow) -> Result { - let mut args = HashMap::new(); - args.insert("event_type".to_string(), Box::new(headers.get("x-event-type").cloned()) as _); - args.insert("payload".to_string(), Box::new(serde_json::from_str::(&body)?) as _); - Ok(PushArgsOwned { extra: None, args }) -} -``` - -Then register in `prepare_native_trigger_args()` in `lib.rs`: -```rust -pub async fn prepare_native_trigger_args(service_name, db, w_id, headers, body) -> Result> { - match service_name { - ServiceName::Google => { /* ... */ Ok(Some(args)) } - ServiceName::NewService => { /* ... */ Ok(Some(args)) } - ServiceName::Nextcloud => Ok(None), // Uses default body parsing - } -} -``` - -### Instance-Level OAuth Credentials - -When `workspace_integrations.oauth_data.instance_shared == true`, `decrypt_oauth_data()` reads `client_id` and `client_secret` from instance-level global settings instead of workspace-level. This allows admins to share OAuth app credentials across workspaces. - -The frontend handles this via the `generate_instance_connect_url` endpoint in `workspace_integrations.rs`. - ---- - -## Testing Checklist - -- [ ] Database migration runs successfully -- [ ] `cargo check -p windmill-native-triggers --features native_trigger` passes -- [ ] `npx svelte-check --threshold error` passes (in frontend/) -- [ ] Service appears in workspace integrations list -- [ ] OAuth flow completes successfully -- [ ] Can create a new trigger -- [ ] Can view trigger details -- [ ] Can update trigger configuration -- [ ] Can delete trigger -- [ ] Webhook receives and processes payloads -- [ ] Background sync works correctly (reconciliation or channel renewal) -- [ ] Error handling works (expired tokens, service unavailable) - ---- - -## Reference Implementations - -### Nextcloud (Self-Hosted, Update+Get Pattern) - -| File | Purpose | -|------|---------| -| `nextcloud/mod.rs` | Types: NextCloudOAuthData, NextcloudServiceConfig, NextCloudTriggerData | -| `nextcloud/external.rs` | External trait: uses update+get pattern, reconcile_with_external_state for sync | -| `nextcloud/routes.rs` | Additional route: `GET /events` | - -Key patterns: relative OAuth endpoints, base_url required, list_all + reconcile for sync, update returns JSON from get(). - -### Google (Cloud, Unified Service, Short Create) - -| File | Purpose | -|------|---------| -| `google/mod.rs` | Types: GoogleServiceConfig with trigger_type discriminator, GoogleTriggerType enum | -| `google/external.rs` | External trait: overrides service_config_from_create_response, channel renewal for sync | -| `google/routes.rs` | Additional routes: `GET /calendars`, `GET /drive/files`, `GET /drive/shared_drives` | - -Key patterns: absolute OAuth endpoints, empty base_url, trigger_type for Drive/Calendar, expiring watch channels with renewal, service_config_from_create_response skips update+get, get() reconstructs data from stored service_config (no external "get channel" API). diff --git a/.claude/skills/native-trigger/SKILL.md b/.claude/skills/native-trigger/SKILL.md new file mode 120000 index 0000000000..18548efdba --- /dev/null +++ b/.claude/skills/native-trigger/SKILL.md @@ -0,0 +1 @@ +../../../.agents/skills/native-trigger/SKILL.md \ No newline at end of file diff --git a/.claude/skills/pr/SKILL.md b/.claude/skills/pr/SKILL.md deleted file mode 100644 index 2c7bd691ca..0000000000 --- a/.claude/skills/pr/SKILL.md +++ /dev/null @@ -1,111 +0,0 @@ ---- -name: pr -user_invocable: true -description: Open a draft pull request on GitHub. MUST use when you want to create/open a PR. ---- - -# Pull Request Skill - -Create a draft pull request with a clear title and explicit description of changes. - -## Instructions - -1. **Analyze branch changes**: Understand all commits since diverging from main -2. **Push to remote**: Ensure all commits are pushed -3. **Create draft PR**: Always open as draft for review before merging - -## PR Title Format - -Follow conventional commit format for the PR title: -``` -: -``` - -### Types -- `feat`: New feature or capability -- `fix`: Bug fix -- `refactor`: Code restructuring -- `docs`: Documentation changes -- `chore`: Maintenance tasks -- `perf`: Performance improvements - -### Title Rules -- Keep under 70 characters -- Use lowercase, imperative mood -- No period at the end -- If `*_ee.rs` files were modified, prefix with `[ee]`: `[ee] : ` - -## PR Body Format - -The body MUST be explicit about what changed. Structure: - -```markdown -## Summary - - -## Changes -- -- -- - -## Test plan -- [ ] -- [ ] - ---- -Generated with [Claude Code](https://claude.com/claude-code) -``` - -## Execution Steps - -1. Run `git status` to check for uncommitted changes -2. Run `git log main..HEAD --oneline` to see all commits in this branch -3. Run `git diff main...HEAD` to see the full diff against main -4. **Run `/local-review`** before creating the PR. If issues are found, fix them and commit before proceeding. Do not skip this step. -5. Check if remote branch exists and is up to date: - ```bash - git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "no upstream" - ``` -6. Push to remote if needed: `git push -u origin HEAD` -7. Create draft PR using gh CLI: - ```bash - gh pr create --draft --title ": " --body "$(cat <<'EOF' - ## Summary - - - ## Changes - - - - - - ## Test plan - - [ ] - - [ ] - - --- - Generated with [Claude Code](https://claude.com/claude-code) - EOF - )" - ``` -8. Return the PR URL to the user - -## EE Companion PR (when `*_ee.rs` files were modified) - -The `*_ee.rs` files in the windmill repo are **symlinks** to `windmill-ee-private` — changes won't appear in `git diff` of the windmill repo. Instead, check the EE repo for uncommitted or unpushed changes. - -Follow the full EE PR workflow in `docs/enterprise.md`. The key PR-specific details: - -1. Find the EE repo/worktree: see "Finding the EE Repo" in `docs/enterprise.md` -2. Check for changes: `git -C status --short` - - If there are no changes in the EE repo, skip this entire section -3. Follow steps 1–5 from the "EE PR Workflow" in `docs/enterprise.md` -4. Create the companion PR (title does NOT get the `[ee]` prefix): - ```bash - gh pr create --draft --repo windmill-labs/windmill-ee-private --title ": " --body "$(cat <<'EOF' - Companion PR for windmill-labs/windmill# - - --- - Generated with [Claude Code](https://claude.com/claude-code) - EOF - )" - ``` -5. Commit `ee-repo-ref.txt` and push the updated windmill branch diff --git a/.claude/skills/pr/SKILL.md b/.claude/skills/pr/SKILL.md new file mode 120000 index 0000000000..9458ad7097 --- /dev/null +++ b/.claude/skills/pr/SKILL.md @@ -0,0 +1 @@ +../../../.agents/skills/pr/SKILL.md \ No newline at end of file diff --git a/.claude/skills/refine/SKILL.md b/.claude/skills/refine/SKILL.md deleted file mode 100644 index aaf747cd29..0000000000 --- a/.claude/skills/refine/SKILL.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: refine -user_invocable: true -description: End-of-session reflection. Reviews friction encountered during the session and proposes updates to docs/ to capture lessons learned. ---- - -# Refine Skill - -Reflect on the current session and update documentation with lessons learned. - -## Instructions - -1. **Identify friction**: Review what happened in this session: - - Run `git diff main...HEAD --stat` to see what files were touched - - Think about: what was slow, what failed, what required multiple attempts, what information was missing or hard to find - -2. **Read current docs**: Read the docs that were relevant to this session: - - `docs/validation.md` - - `docs/enterprise.md` - - `docs/autonomous-mode.md` - - Any skills that were invoked - -3. **Propose updates**: For each piece of friction, decide if it warrants a doc update: - - **Missing knowledge**: Information you had to discover that should be documented - - **Wrong guidance**: Instructions that led you astray - - **Missing validation rule**: A check that should be in the validation matrix - - **New pattern**: A codebase pattern worth capturing for next time - -4. **Apply updates**: Edit the relevant `docs/` files. Keep changes minimal and specific — add only what would have saved time this session. - -5. **Report**: Summarize what was added/changed and why. - -## Rules - -- Only add knowledge confirmed by this session — no speculative additions -- Keep docs concise — add a line or two, not a paragraph -- If a whole new doc is needed, create it in `docs/` and add a pointer in `CLAUDE.md` -- Don't update skills unless a coding pattern was genuinely wrong -- Don't add things Claude already knows — only Windmill-specific knowledge diff --git a/.claude/skills/refine/SKILL.md b/.claude/skills/refine/SKILL.md new file mode 120000 index 0000000000..39580df5d0 --- /dev/null +++ b/.claude/skills/refine/SKILL.md @@ -0,0 +1 @@ +../../../.agents/skills/refine/SKILL.md \ No newline at end of file diff --git a/.claude/skills/rust-backend/SKILL.md b/.claude/skills/rust-backend/SKILL.md deleted file mode 100644 index f0c52002bc..0000000000 --- a/.claude/skills/rust-backend/SKILL.md +++ /dev/null @@ -1,107 +0,0 @@ ---- -name: rust-backend -description: Rust coding guidelines for the Windmill backend. MUST use when writing or modifying Rust code in the backend directory. ---- - -# Windmill Rust Patterns - -Apply these Windmill-specific patterns when writing Rust code in `backend/`. - -## Error Handling - -Use `Error` from `windmill_common::error`. Return `Result` or `JsonResult`: - -```rust -use windmill_common::error::{Error, Result}; - -pub async fn get_job(db: &DB, id: Uuid) -> Result { - sqlx::query_as!(Job, "SELECT id, workspace_id FROM v2_job WHERE id = $1", id) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::NotFound("job not found".to_string()))?; -} -``` - -Never panic in library code. Reserve `.unwrap()` for compile-time guarantees. - -## SQLx Patterns - -**Never use `SELECT *`** — always list columns explicitly. Critical for backwards compatibility when workers lag behind API version: - -```rust -// Correct -sqlx::query_as!(Job, "SELECT id, workspace_id, path FROM v2_job WHERE id = $1", id) - -// Wrong — breaks when columns are added -sqlx::query_as!(Job, "SELECT * FROM v2_job WHERE id = $1", id) -``` - -Use batch operations to avoid N+1: - -```rust -// Preferred — single query with IN clause -sqlx::query!("SELECT ... WHERE id = ANY($1)", &ids[..]).fetch_all(db).await? -``` - -Use transactions for multi-step operations. Parameterize all queries. - -## JSON Handling - -Prefer `Box` over `serde_json::Value` when storing/passing JSON without inspection: - -```rust -pub struct Job { - pub args: Option>, -} -``` - -Only use `serde_json::Value` when you need to inspect or modify the JSON. - -## Serde Optimizations - -```rust -#[derive(Serialize, Deserialize)] -pub struct Job { - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_job: Option, - #[serde(skip_serializing_if = "Vec::is_empty")] - pub tags: Vec, - #[serde(default)] - pub priority: i32, -} -``` - -## Async & Concurrency - -Never block the async runtime. Use `spawn_blocking` for CPU-intensive work: - -```rust -let result = tokio::task::spawn_blocking(move || expensive_computation(&data)).await?; -``` - -**Mutex selection**: Prefer `std::sync::Mutex` (or `parking_lot::Mutex`) for data protection. Only use `tokio::sync::Mutex` when holding locks across `.await` points. - -Use `tokio::sync::mpsc` (bounded) for channels. Avoid `std::thread::sleep` in async contexts. - -## Module Structure & Visibility - -- Use `pub(crate)` instead of `pub` when possible -- Place new code in the appropriate crate based on functionality -- API endpoints go in `windmill-api/src/` organized by domain -- Shared functionality goes in `windmill-common/src/` - -## Code Navigation - -Always use rust-analyzer LSP for go-to-definition, find-references, and type info. Do not guess at module paths. - -## Axum Handlers - -Destructure extractors directly in function signatures: - -```rust -async fn process_job( - Extension(db): Extension, - Path((workspace, job_id)): Path<(String, Uuid)>, - Query(pagination): Query, -) -> Result> { ... } -``` diff --git a/.claude/skills/rust-backend/SKILL.md b/.claude/skills/rust-backend/SKILL.md new file mode 120000 index 0000000000..2500c55046 --- /dev/null +++ b/.claude/skills/rust-backend/SKILL.md @@ -0,0 +1 @@ +../../../.agents/skills/rust-backend/SKILL.md \ No newline at end of file diff --git a/.claude/skills/svelte-frontend/SKILL.md b/.claude/skills/svelte-frontend/SKILL.md deleted file mode 100644 index 57cac70302..0000000000 --- a/.claude/skills/svelte-frontend/SKILL.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -name: svelte-frontend -description: Svelte coding guidelines for the Windmill frontend. MUST use when writing or modifying code in the frontend directory. ---- - -# Windmill Svelte Patterns - -Apply these Windmill-specific patterns when writing Svelte code in `frontend/`. For general Svelte 5 syntax (runes, snippets, event handling), use the Svelte MCP server. - -## Windmill UI Components (MUST use) - -Always use Windmill's design-system components. Never use raw HTML elements. - -### Buttons — ` - +
+ {/each} + +{/if} diff --git a/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts b/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts new file mode 100644 index 0000000000..725385d3ee --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts @@ -0,0 +1,40 @@ +import { sendUserToast } from '$lib/toast' +import type { ToolDisplayAction } from './shared' + +type MaybePromise = T | Promise +type ToolDisplayActionHandler = (action: ToolDisplayAction) => MaybePromise + +const toolDisplayActionHandlers = $state>({}) + +function formatUnknownError(error: unknown): string { + if (error instanceof Error) { + return error.message + } + return String(error) +} + +export function registerToolDisplayActionHandler( + type: ToolDisplayAction['type'], + handler: ToolDisplayActionHandler +): () => void { + toolDisplayActionHandlers[type] = handler + return () => { + if (toolDisplayActionHandlers[type] === handler) { + delete toolDisplayActionHandlers[type] + } + } +} + +export async function runToolDisplayAction(action: ToolDisplayAction): Promise { + const handler = toolDisplayActionHandlers[action.type] + if (!handler) { + sendUserToast('This action is not available right now.', true) + return + } + + try { + await handler(action) + } catch (error) { + sendUserToast(`Could not run action "${action.label}": ${formatUnknownError(error)}`, true) + } +} diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 76b1503225..1cb0b3edc0 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -277,7 +277,17 @@ describe('processToolCall', () => { target_path: 'f/scripts/current', target_kind: 'script', backend_result: 'schedule-created' - }) + }), + actions: [ + expect.objectContaining({ + id: 'open-created-schedule:f/schedules/current', + type: 'open_created_resource', + label: 'Open schedule', + resource: 'schedule', + path: 'f/schedules/current', + targetKind: 'script' + }) + ] }) ) expect(JSON.parse(scheduleResult.content as string)).toEqual( @@ -342,7 +352,18 @@ describe('processToolCall', () => { target_path: 'f/flows/current', target_kind: 'flow', backend_result: 'trigger-created' - }) + }), + actions: [ + expect.objectContaining({ + id: 'open-created-trigger:http:f/triggers/current', + type: 'open_created_resource', + label: 'Open HTTP trigger', + resource: 'trigger', + triggerKind: 'http', + path: 'f/triggers/current', + targetKind: 'flow' + }) + ] }) ) expect(JSON.parse(triggerResult.content as string)).toEqual( diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 521c25fdaf..59eba6a834 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -409,6 +409,29 @@ export type UserDisplayMessage = BaseDisplayMessage & { error?: boolean } +export type CreatedResourceTriggerKind = + | 'http' + | 'websocket' + | 'kafka' + | 'nats' + | 'postgres' + | 'mqtt' + | 'sqs' + | 'gcp' + | 'azure' + +export type CreatedResourceAction = { + id: string + type: 'open_created_resource' + label: string + resource: 'schedule' | 'trigger' + path: string + targetKind: 'script' | 'flow' + triggerKind?: CreatedResourceTriggerKind +} + +export type ToolDisplayAction = CreatedResourceAction + export type ToolDisplayMessage = { role: 'tool' tool_call_id: string @@ -423,6 +446,7 @@ export type ToolDisplayMessage = { isStreamingArguments?: boolean toolName?: string showFade?: boolean + actions?: ToolDisplayAction[] } export type AssistantDisplayMessage = BaseDisplayMessage & { diff --git a/frontend/src/lib/components/copilot/chat/workspaceTools.ts b/frontend/src/lib/components/copilot/chat/workspaceTools.ts index 37b98482f1..910b7ccbc7 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceTools.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceTools.ts @@ -26,7 +26,13 @@ import { triggerRequestSchemas } from './workspaceToolsZod.gen' import { z } from 'zod' -import { createToolDef, type Tool, type ToolCallbacks } from './shared' +import { + createToolDef, + type CreatedResourceTriggerKind, + type Tool, + type ToolCallbacks, + type ToolDisplayAction +} from './shared' import { emptyString } from '$lib/utils' type TriggerKind = keyof typeof triggerRequestSchemas @@ -60,7 +66,9 @@ function getWorkspaceMutationTarget(helpers: unknown): WorkspaceMutationTarget | return (helpers as WorkspaceMutationHelpers | undefined)?.getWorkspaceMutationTarget?.() } -function getWorkspaceMutationTargetError(target: WorkspaceMutationTarget | undefined): string | undefined { +function getWorkspaceMutationTargetError( + target: WorkspaceMutationTarget | undefined +): string | undefined { if (!target) { return 'the script or flow needs to be deployed before doing this action' } @@ -74,7 +82,9 @@ function validateWorkspaceMutationTarget(helpers: unknown): string | undefined { return getWorkspaceMutationTargetError(getWorkspaceMutationTarget(helpers)) } -function requireWorkspaceMutationTarget(helpers: unknown): WorkspaceMutationTarget & { path: string } { +function requireWorkspaceMutationTarget( + helpers: unknown +): WorkspaceMutationTarget & { path: string } { const target = getWorkspaceMutationTarget(helpers) const error = getWorkspaceMutationTargetError(target) if (error) { @@ -83,7 +93,9 @@ function requireWorkspaceMutationTarget(helpers: unknown): WorkspaceMutationTarg return target as WorkspaceMutationTarget & { path: string } } -function getWorkspaceMutationTargetFields(helpers: unknown): Pick { +function getWorkspaceMutationTargetFields( + helpers: unknown +): Pick { const target = requireWorkspaceMutationTarget(helpers) return { script_path: target.path, @@ -168,6 +180,38 @@ const triggerConfigs = { } } +function getActionTargetKind(isFlow: boolean): 'script' | 'flow' { + return isFlow ? 'flow' : 'script' +} + +function createOpenScheduleAction(path: string, targetKind: 'script' | 'flow'): ToolDisplayAction { + return { + id: `open-created-schedule:${path}`, + type: 'open_created_resource', + label: 'Open schedule', + resource: 'schedule', + path, + targetKind + } +} + +function createOpenTriggerAction( + kind: TriggerKind, + path: string, + targetKind: 'script' | 'flow', + label: string +): ToolDisplayAction { + return { + id: `open-created-trigger:${kind}:${path}`, + type: 'open_created_resource', + label: `Open ${label}`, + resource: 'trigger', + triggerKind: kind as CreatedResourceTriggerKind, + path, + targetKind + } +} + function formatPath(path: (string | number | symbol)[]): string { if (path.length === 0) { return 'value' @@ -265,16 +309,18 @@ const createScheduleTool: Tool = { }) try { const result = await ScheduleService.createSchedule({ workspace, requestBody }) + const targetKind = getActionTargetKind(requestBody.is_flow) const toolResult = { success: true, path: requestBody.path, target_path: requestBody.script_path, - target_kind: requestBody.is_flow ? 'flow' : 'script', + target_kind: targetKind, backend_result: result } toolCallbacks.setToolStatus(toolId, { content: `Created schedule "${requestBody.path}"`, - result: toolResult + result: toolResult, + actions: [createOpenScheduleAction(requestBody.path, targetKind)] }) return JSON.stringify(toolResult) } catch (error) { @@ -311,17 +357,26 @@ const createTriggerTool: Tool = { }) try { const result = await triggerConfig.create({ workspace, requestBody } as never) + const targetKind = getActionTargetKind(requestBody.is_flow) const toolResult = { success: true, kind: parsedArgs.kind, path: requestBody.path, target_path: requestBody.script_path, - target_kind: requestBody.is_flow ? 'flow' : 'script', + target_kind: targetKind, backend_result: result } toolCallbacks.setToolStatus(toolId, { content: `Created ${triggerConfig.label} "${requestBody.path}"`, - result: toolResult + result: toolResult, + actions: [ + createOpenTriggerAction( + parsedArgs.kind, + requestBody.path, + targetKind, + triggerConfig.label + ) + ] }) return JSON.stringify(toolResult) } catch (error) { From b86f8960fcd8a66bc6849638178ef45ac49e06f1 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 5 May 2026 20:39:06 +0200 Subject: [PATCH 29/56] fix(windmill-utils-internal): move config to subpath export (#9045) The config module imports node:fs/promises (stat, mkdir), which breaks non-Node bundlers like the Cloudflare Workers build of the hub. The windmill SPA frontend got away with it via tree-shaking, but stricter runtimes choke on the bare node: import even when unused. Stop re-exporting ./config from the main entry and expose it via a windmill-utils-internal/config subpath instead. CLI code already deep-imports the source file, so it is unaffected. Bumps the package to 1.4.0 and updates the frontend dependency to match. Co-authored-by: Claude Opus 4.7 (1M context) --- cli/windmill-utils-internal/package-lock.json | 4 +- cli/windmill-utils-internal/package.json | 7 ++- cli/windmill-utils-internal/src/index.ts | 1 - frontend/package-lock.json | 58 ++----------------- frontend/package.json | 2 +- 5 files changed, 15 insertions(+), 57 deletions(-) diff --git a/cli/windmill-utils-internal/package-lock.json b/cli/windmill-utils-internal/package-lock.json index e9ce5e3c9a..36a8962fa9 100644 --- a/cli/windmill-utils-internal/package-lock.json +++ b/cli/windmill-utils-internal/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-utils-internal", - "version": "1.3.6", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-utils-internal", - "version": "1.3.6", + "version": "1.4.0", "license": "Apache 2.0", "devDependencies": { "@types/node": "^24.2.0", diff --git a/cli/windmill-utils-internal/package.json b/cli/windmill-utils-internal/package.json index 56dcfd9905..a0691c56c3 100644 --- a/cli/windmill-utils-internal/package.json +++ b/cli/windmill-utils-internal/package.json @@ -1,6 +1,6 @@ { "name": "windmill-utils-internal", - "version": "1.3.8", + "version": "1.4.0", "description": "Internal utility functions for Windmill", "main": "dist/cjs/index.js", "module": "dist/esm/index.js", @@ -10,6 +10,11 @@ "require": "./dist/cjs/index.js", "import": "./dist/esm/index.js", "types": "./dist/esm/index.d.ts" + }, + "./config": { + "require": "./dist/cjs/config/index.js", + "import": "./dist/esm/config/index.js", + "types": "./dist/esm/config/index.d.ts" } }, "scripts": { diff --git a/cli/windmill-utils-internal/src/index.ts b/cli/windmill-utils-internal/src/index.ts index 52c489da4c..d23a11e8a6 100644 --- a/cli/windmill-utils-internal/src/index.ts +++ b/cli/windmill-utils-internal/src/index.ts @@ -11,6 +11,5 @@ export * from "./inline-scripts"; export * from "./path-utils"; export * from "./parse"; -export * from "./config"; export * from "./deploy"; export { SEP, DELIMITER } from "./constants"; \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 767365375d..6fcd7f34c0 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -90,7 +90,7 @@ "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.3.8", + "windmill-utils-internal": "^1.4.0", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", @@ -844,7 +844,6 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.0.tgz", "integrity": "sha512-0DQ98G9ZQZOxfUcQn1waV2yS8aWdZ6kJMbYCJB3oUBecjWYO1fqJ+a1DRfPF3O5JEkwqwP1A9QEN/9mYm2Yd0w==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -856,7 +855,6 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.0.tgz", "integrity": "sha512-QN75eB0IH2ywSpRpNddCRfQIhmJYBCJ1x5Lb3IscKAL8bMnVAKnRg8dCoXbHzVLLH7P38N2Z3mtulB7W0J0FKw==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -867,7 +865,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.0.tgz", "integrity": "sha512-N10dEJNSsUx41Z6pZsXU8FjPjpBEplgH24sfkmITrBED1/U2Esum9F3lfLrMjKHHjmi557zQn7kR9R+XWXu5Rg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1357,7 +1354,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz", "integrity": "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1514,7 +1510,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1531,7 +1526,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1548,7 +1542,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1565,7 +1558,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1582,7 +1574,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1599,7 +1590,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1616,7 +1606,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1633,7 +1622,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1650,7 +1638,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1667,7 +1654,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1684,7 +1670,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1701,7 +1686,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1718,7 +1702,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1735,7 +1718,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1752,7 +1734,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2058,7 +2039,6 @@ "version": "0.10.1", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -6834,7 +6814,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "dev": true, + "devOptional": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -7333,7 +7313,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7354,7 +7333,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7375,7 +7353,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7396,7 +7373,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7417,7 +7393,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7438,7 +7413,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7459,7 +7433,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7480,7 +7453,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7501,7 +7473,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7522,7 +7493,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7543,7 +7513,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -12112,21 +12081,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -12857,7 +12811,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -13706,9 +13660,9 @@ "integrity": "sha512-uHNL8F72/Tf96xF3hOHnPDjkEyqXw7fNjcPJiUhth9sTQkcwUIoJMOdwm8/cs+j9kKVRJ4tgNYMHEBLylazp6g==" }, "node_modules/windmill-utils-internal": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.3.8.tgz", - "integrity": "sha512-FtVEvAI2PIqPTEpowTjo5c5JkYe09Scu9zcwzJutOWMEh4aDdzOejaG7EZTac0pk+dK4JB46+nbl82hhLsL8Mw==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.4.0.tgz", + "integrity": "sha512-IX5MEuHkRTDyqtCDh6Qez43faAD8Q7VcFTDriZZ831l3ZPvBZSch5D0ejeu0DLc1GE0GOj4YoPXzt1mVO8GFBA==", "license": "Apache 2.0" }, "node_modules/word-wrap": { diff --git a/frontend/package.json b/frontend/package.json index ff700a5fce..508e3813f8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -163,7 +163,7 @@ "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.3.8", + "windmill-utils-internal": "^1.4.0", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", From 6e5a21a9c7b5db77d325b5916a9ab8799a2eb6e7 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 5 May 2026 22:11:38 +0200 Subject: [PATCH 30/56] fix(flows): inherit flow_env in sub-flow predicates (#9042) * fix(flows): inherit flow_env in sub-flow predicates Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(flows): align flow_env lookup with get_root_job_id and tighten gate Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(flows): drop recursive CTE, root_job propagation suffices Co-Authored-By: Claude Opus 4.7 (1M context) * fix(flows): walk via flow_innermost_root_job to respect imported-flow scope Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(flows): remove flow_env API endpoint, dead code from deno_core era Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- ...5a9108b6f32490cb61b14de71102bb305f71d.json | 24 + ...86d7c4b9bdb1e8fb1f7725060990ef8984943.json | 24 - ...85242186449ff592ed457daac59b37b94aa00.json | 17 + ...3f5343e68856dfede19597813893b7e99ead1.json | 17 + backend/tests/flow_engine_parity.rs | 1046 +++++++++++++++++ backend/windmill-api/src/jobs.rs | 128 -- backend/windmill-common/src/client.rs | 20 - backend/windmill-jseval/src/lib.rs | 54 +- backend/windmill-worker/src/worker_flow.rs | 160 ++- 9 files changed, 1281 insertions(+), 209 deletions(-) create mode 100644 backend/.sqlx/query-14c2784a68f06e7349941671abe5a9108b6f32490cb61b14de71102bb305f71d.json delete mode 100644 backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json create mode 100644 backend/.sqlx/query-54d3dd91f3348c03b8b39ecb59c85242186449ff592ed457daac59b37b94aa00.json create mode 100644 backend/.sqlx/query-ea132d80fdb6525f192797fd77b3f5343e68856dfede19597813893b7e99ead1.json diff --git a/backend/.sqlx/query-14c2784a68f06e7349941671abe5a9108b6f32490cb61b14de71102bb305f71d.json b/backend/.sqlx/query-14c2784a68f06e7349941671abe5a9108b6f32490cb61b14de71102bb305f71d.json new file mode 100644 index 0000000000..d7ef4e04e0 --- /dev/null +++ b/backend/.sqlx/query-14c2784a68f06e7349941671abe5a9108b6f32490cb61b14de71102bb305f71d.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE chain(id, parent_job, flow_innermost_root_job, runnable_id, runnable_path, raw_flow, depth) AS (\n SELECT id, parent_job, flow_innermost_root_job, runnable_id, runnable_path, raw_flow, 0\n FROM v2_job\n WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job, j.flow_innermost_root_job, j.runnable_id, j.runnable_path, j.raw_flow, c.depth + 1\n FROM v2_job j\n JOIN chain c\n ON j.id = COALESCE(c.flow_innermost_root_job, c.parent_job)\n WHERE j.workspace_id = $2 AND c.depth < $3\n )\n SELECT\n CASE\n WHEN flow_version.id IS NOT NULL THEN\n flow_version.value -> 'flow_env'\n ELSE\n chain.raw_flow -> 'flow_env'\n END AS \"flow_env: Json>>\"\n FROM chain\n LEFT JOIN flow_version\n ON flow_version.id = chain.runnable_id\n AND flow_version.path = chain.runnable_path\n AND flow_version.workspace_id = $2\n WHERE (CASE\n WHEN flow_version.id IS NOT NULL THEN flow_version.value -> 'flow_env'\n ELSE chain.raw_flow -> 'flow_env'\n END) IS NOT NULL\n ORDER BY chain.depth ASC\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flow_env: Json>>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int4" + ] + }, + "nullable": [ + null + ] + }, + "hash": "14c2784a68f06e7349941671abe5a9108b6f32490cb61b14de71102bb305f71d" +} diff --git a/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json b/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json deleted file mode 100644 index 8c5f43ab07..0000000000 --- a/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n CASE\n WHEN flow_version.id IS NOT NULL THEN\n flow_version.value -> 'flow_env' -> $3\n ELSE\n root_job.raw_flow -> 'flow_env' -> $3\n END AS \"flow_env: sqlx::types::Json>\"\n FROM\n v2_job current_job\n JOIN\n v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id)\n AND root_job.workspace_id = current_job.workspace_id\n LEFT JOIN\n flow_version ON flow_version.id = root_job.runnable_id\n AND flow_version.path = root_job.runnable_path\n AND flow_version.workspace_id = root_job.workspace_id\n WHERE\n current_job.id = $1 AND\n current_job.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "flow_env: sqlx::types::Json>", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943" -} diff --git a/backend/.sqlx/query-54d3dd91f3348c03b8b39ecb59c85242186449ff592ed457daac59b37b94aa00.json b/backend/.sqlx/query-54d3dd91f3348c03b8b39ecb59c85242186449ff592ed457daac59b37b94aa00.json new file mode 100644 index 0000000000..962446787e --- /dev/null +++ b/backend/.sqlx/query-54d3dd91f3348c03b8b39ecb59c85242186449ff592ed457daac59b37b94aa00.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ($1, '', '', $2, ARRAY[$3]::bigint[], '{}'::jsonb, $4, 'system')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "54d3dd91f3348c03b8b39ecb59c85242186449ff592ed457daac59b37b94aa00" +} diff --git a/backend/.sqlx/query-ea132d80fdb6525f192797fd77b3f5343e68856dfede19597813893b7e99ead1.json b/backend/.sqlx/query-ea132d80fdb6525f192797fd77b3f5343e68856dfede19597813893b7e99ead1.json new file mode 100644 index 0000000000..593408b1d4 --- /dev/null +++ b/backend/.sqlx/query-ea132d80fdb6525f192797fd77b3f5343e68856dfede19597813893b7e99ead1.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) VALUES ($1, $2, $3, '{}'::jsonb, $4, 'system')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Varchar", + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ea132d80fdb6525f192797fd77b3f5343e68856dfede19597813893b7e99ead1" +} diff --git a/backend/tests/flow_engine_parity.rs b/backend/tests/flow_engine_parity.rs index 363ab4fb16..bae776e7f1 100644 --- a/backend/tests/flow_engine_parity.rs +++ b/backend/tests/flow_engine_parity.rs @@ -2097,3 +2097,1049 @@ export function main( Ok(()) } + +// ============================================================================= +// flow_env inside predicates of nested sub-flows (BranchOne / loops). +// +// Sub-flows spawned by `payload_from_modules` for branches/loops don't carry +// the parent's `flow_env` in their own FlowValue, so without explicit lookup +// the predicate evaluators receive `None` and `flow_env.X` resolves to +// `undefined` inside QuickJS. Verify `handle_flow` walks up to the nearest +// enclosing scope so predicates see the inherited env. +// ============================================================================= + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_env_skip_if_inside_branchone(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + let mut flow_env = std::collections::HashMap::new(); + flow_env.insert( + "SKIP".to_string(), + windmill_common::worker::to_raw_value(&json!(true)), + ); + + let inner_step = { + let mut m = flow_module( + "inner", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" +export function main() { + return {ran: true}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + // skip_if uses `Boolean(...)`-wrapped expression, so it falls through + // to QuickJS and exercises the local flow_env propagation path. + m.skip_if = + Some(windmill_common::flows::SkipIf { expr: "flow_env.SKIP === true".to_string() }); + m + }; + + // Branch with two modules: a marker that runs first, then `inner` which + // should be skipped via `flow_env.SKIP === true`. When skipped, `inner` + // becomes an identity job and the branch's terminal result is whatever + // `previous_result` was at that point — i.e. the branch_marker output. + let branch_marker = flow_module( + "branch_marker", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" +export function main() { + return {marker: "branch-marker"}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + + let flow = FlowValue { + modules: vec![ + flow_module( + "router", + FlowModuleValue::BranchOne { + branches: vec![Branch { + summary: None, + expr: "true".to_string(), + modules: vec![branch_marker, inner_step], + modules_node: None, + skip_failure: false, + parallel: false, + }], + default: vec![], + default_node: None, + }, + ), + flow_module( + "after", + FlowModuleValue::RawScript { + input_transforms: [js_input("prev", "previous_result")].into(), + language: ScriptLang::Deno, + content: r#" +export function main(prev: any) { + return {prev}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), + ], + flow_env: Some(flow_env), + same_worker: false, + ..Default::default() + }; + + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); + + // With the fix, skip_if sees `flow_env.SKIP === true`, `inner` becomes an + // identity step and passes through `previous_result` (branch_marker). + // Without the fix, flow_env was None inside the sub-flow, the predicate + // returned false, and `inner` ran, leaving `{ran: true}` in `prev`. + assert_eq!( + result["prev"]["marker"], "branch-marker", + "skip_if with flow_env should skip `inner`; expected branch_marker passed through (got {result:?})" + ); + assert!( + result["prev"].get("ran").is_none(), + "`inner` ran when it should have been skipped (got {result:?})" + ); + + Ok(()) +} + +// Nested sub-flows: branch inside branch. The recursive CTE in +// `fetch_root_flow_env` must walk past more than one layer of +// `payload_from_modules`-constructed FlowValue (each of which has +// `flow_env = None`) to reach the root's flow_env. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_env_skip_if_nested_branchone(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + let mut flow_env = std::collections::HashMap::new(); + flow_env.insert( + "SKIP".to_string(), + windmill_common::worker::to_raw_value(&json!(true)), + ); + + let leaf = { + let mut m = flow_module( + "leaf", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" +export function main() { + return {ran: true}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + m.skip_if = + Some(windmill_common::flows::SkipIf { expr: "flow_env.SKIP === true".to_string() }); + m + }; + + let inner_marker = flow_module( + "inner_marker", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" +export function main() { + return {marker: "inner"}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + + // Inner BranchOne: contains the marker + the leaf with skip_if. + let inner_branch = flow_module( + "inner_router", + FlowModuleValue::BranchOne { + branches: vec![Branch { + summary: None, + expr: "true".to_string(), + modules: vec![inner_marker, leaf], + modules_node: None, + skip_failure: false, + parallel: false, + }], + default: vec![], + default_node: None, + }, + ); + + // Outer BranchOne: contains the inner BranchOne. So the leaf is two + // levels deep in payload_from_modules-constructed sub-flows. + let outer = flow_module( + "outer_router", + FlowModuleValue::BranchOne { + branches: vec![Branch { + summary: None, + expr: "true".to_string(), + modules: vec![inner_branch], + modules_node: None, + skip_failure: false, + parallel: false, + }], + default: vec![], + default_node: None, + }, + ); + + let after = flow_module( + "after", + FlowModuleValue::RawScript { + input_transforms: [js_input("prev", "previous_result")].into(), + language: ScriptLang::Deno, + content: r#" +export function main(prev: any) { + return {prev}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + + let flow = FlowValue { + modules: vec![outer, after], + flow_env: Some(flow_env), + same_worker: false, + ..Default::default() + }; + + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); + + // Leaf's skip_if should see flow_env.SKIP=true → leaf becomes identity → + // previous_result inside the inner branch is `inner_marker`. That bubbles + // up to the outer branch and into `after`. + assert_eq!( + result["prev"]["marker"], "inner", + "skip_if with flow_env should skip `leaf` even nested two layers deep (got {result:?})" + ); + assert!( + result["prev"].get("ran").is_none(), + "`leaf` ran when it should have been skipped two layers deep (got {result:?})" + ); + + Ok(()) +} + +// Complex input-transform expression inside a sub-flow exercises the +// QuickJS evaluation path (it doesn't match the `flow_env.X` / +// `flow_env.X.Y` regex that hits the API fast path). Without flow_env +// inheritance, QuickJS would see an empty `flow_env` and the expression +// would NaN/undefined out. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_env_complex_input_transform_in_branch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + let mut flow_env = std::collections::HashMap::new(); + flow_env.insert( + "LIMIT".to_string(), + windmill_common::worker::to_raw_value(&json!(7)), + ); + flow_env.insert( + "OFFSET".to_string(), + windmill_common::worker::to_raw_value(&json!(3)), + ); + + let inner = flow_module( + "compute", + FlowModuleValue::RawScript { + // Expression doesn't match the regex fast path (uses arithmetic + // and Math.min), so the worker falls through to QuickJS using + // the local flow_env. Without inheritance, this is empty. + input_transforms: [js_input( + "value", + "Math.min(flow_env.LIMIT, 10) + flow_env.OFFSET", + )] + .into(), + language: ScriptLang::Deno, + content: r#" +export function main(value: number) { + return {value}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + + let flow = FlowValue { + modules: vec![flow_module( + "router", + FlowModuleValue::BranchOne { + branches: vec![Branch { + summary: None, + expr: "true".to_string(), + modules: vec![inner], + modules_node: None, + skip_failure: false, + parallel: false, + }], + default: vec![], + default_node: None, + }, + )], + flow_env: Some(flow_env), + same_worker: false, + ..Default::default() + }; + + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); + + // min(7, 10) + 3 = 10 + assert_eq!( + result["value"], 10, + "complex input transform `Math.min(flow_env.LIMIT, 10) + flow_env.OFFSET` should resolve via QuickJS with inherited flow_env (got {result:?})" + ); + + Ok(()) +} + +// Parallel for-loop iterations are pushed with `flow_innermost_root_job = +// None` (worker_flow.rs:3941), so the recursive CTE in `fetch_root_flow_env` +// must use `parent_job` to walk up. Verify a skip_if inside an iteration +// sub-flow sees the parent's flow_env. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_env_skip_if_in_parallel_forloop(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + let mut flow_env = std::collections::HashMap::new(); + flow_env.insert( + "SKIP".to_string(), + windmill_common::worker::to_raw_value(&json!(true)), + ); + + let inner_marker = flow_module( + "marker", + FlowModuleValue::RawScript { + input_transforms: [js_input("i", "flow_input.iter.value")].into(), + language: ScriptLang::Deno, + content: r#" +export function main(i: number) { + return {marker: i}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + + let leaf = { + let mut m = flow_module( + "leaf", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" +export function main() { + return {ran: true}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + m.skip_if = + Some(windmill_common::flows::SkipIf { expr: "flow_env.SKIP === true".to_string() }); + m + }; + + let flow = FlowValue { + modules: vec![flow_module( + "loop", + FlowModuleValue::ForloopFlow { + iterator: InputTransform::Javascript { expr: "[1, 2]".to_string() }, + modules: vec![inner_marker, leaf], + modules_node: None, + skip_failures: false, + parallel: true, + parallelism: None, + squash: None, + }, + )], + flow_env: Some(flow_env), + same_worker: false, + ..Default::default() + }; + + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); + + // Each iteration's `leaf` is skipped (skip_if reads flow_env.SKIP=true via + // parent_job lookup since parallel iterations have flow_innermost_root_job + // = None). The skipped step passes through previous_result = marker's + // output. So iteration result = `{marker: i}`, not `{ran: true}`. + let arr = result.as_array().expect("parallel loop result is an array"); + assert_eq!(arr.len(), 2, "expected 2 iterations, got {result:?}"); + for (i, iter_result) in arr.iter().enumerate() { + assert_eq!( + iter_result["marker"], + json!(i + 1), + "iteration {i} marker mismatch (got {result:?})" + ); + assert!( + iter_result.get("ran").is_none(), + "leaf ran in iteration {i} when it should have been skipped (got {result:?})" + ); + } + + Ok(()) +} + +// Imported flows (`FlowModuleValue::Flow { path }`) load their value from +// `flow_version`. Two cases: +// (a) the imported flow defines its own flow_env → that wins, parent's is +// NOT merged (current behavior; option (i) per design discussion). +// (b) the imported flow defines no flow_env → it inherits from the parent +// via the recursive CTE. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_env_imported_flow_uses_own_env(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + // Imported flow with its own flow_env: a single step that returns + // `flow_env.KEY`. Saved at f/system/imported_with_env. + let imported_path = "f/system/imported_with_env"; + let imported_value = json!({ + "modules": [{ + "id": "leaf", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(key: string) { return {key}; }", + "input_transforms": { + "key": { "type": "javascript", "expr": "flow_env.KEY" } + } + } + }], + "flow_env": { "KEY": "imported" } + }); + let imported_version_id: i64 = 9991001; + sqlx::query!( + "INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ($1, '', '', $2, ARRAY[$3]::bigint[], '{}'::jsonb, $4, 'system')", + "test-workspace", + imported_path, + imported_version_id, + imported_value.clone(), + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) VALUES ($1, $2, $3, '{}'::jsonb, $4, 'system')", + imported_version_id, + "test-workspace", + imported_path, + imported_value.clone(), + ) + .execute(&db) + .await?; + + let mut parent_env = std::collections::HashMap::new(); + parent_env.insert( + "KEY".to_string(), + windmill_common::worker::to_raw_value(&json!("parent")), + ); + + let parent = FlowValue { + modules: vec![flow_module( + "import", + FlowModuleValue::Flow { + input_transforms: Default::default(), + path: imported_path.to_string(), + pass_flow_input_directly: None, + }, + )], + flow_env: Some(parent_env), + same_worker: false, + ..Default::default() + }; + + let result = + RunJob::from(JobPayload::RawFlow { value: parent, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); + + // The imported flow's `leaf` reads flow_env.KEY. The imported flow has its + // own flow_env so it wins — result should be "imported", not "parent". + assert_eq!( + result["key"], "imported", + "imported flow's own flow_env should win over parent's (got {result:?})" + ); + + Ok(()) +} + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_env_imported_flow_inherits_when_unset(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + // Imported flow without its own flow_env. The leaf step's `skip_if` uses + // a Boolean()-wrapped expression — that always falls through to QuickJS + // (no regex/API fast path) and reads the LOCAL flow_env. Without the fix, + // local flow_env inside the imported sub-flow is None and the predicate + // returns false; with the fix, the imported flow inherits the parent's + // env via the recursive CTE. + let imported_path = "f/system/imported_no_env"; + let imported_value = json!({ + "modules": [ + { + "id": "marker", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return {marker: \"from-imported\"}; }", + "input_transforms": {} + } + }, + { + "id": "leaf", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return {ran: true}; }", + "input_transforms": {} + }, + "skip_if": { "expr": "flow_env.KEY === 'parent'" } + } + ] + }); + let imported_version_id: i64 = 9991002; + sqlx::query!( + "INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ($1, '', '', $2, ARRAY[$3]::bigint[], '{}'::jsonb, $4, 'system')", + "test-workspace", + imported_path, + imported_version_id, + imported_value.clone(), + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) VALUES ($1, $2, $3, '{}'::jsonb, $4, 'system')", + imported_version_id, + "test-workspace", + imported_path, + imported_value.clone(), + ) + .execute(&db) + .await?; + + let mut parent_env = std::collections::HashMap::new(); + parent_env.insert( + "KEY".to_string(), + windmill_common::worker::to_raw_value(&json!("parent")), + ); + + let parent = FlowValue { + modules: vec![flow_module( + "import", + FlowModuleValue::Flow { + input_transforms: Default::default(), + path: imported_path.to_string(), + pass_flow_input_directly: None, + }, + )], + flow_env: Some(parent_env), + same_worker: false, + ..Default::default() + }; + + let result = + RunJob::from(JobPayload::RawFlow { value: parent, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); + + // Imported flow has no flow_env → inherits parent's via lookup. The + // leaf's skip_if (`flow_env.KEY === 'parent'`) evaluates to true → leaf + // becomes identity, passes through `previous_result` (marker's output). + // Without the fix, skip_if's QuickJS context has flow_env=None inside + // the imported sub-flow, the predicate is false, and `leaf` runs. + assert_eq!( + result["marker"], "from-imported", + "imported flow's leaf should be skipped via inherited flow_env (got {result:?})" + ); + assert!( + result.get("ran").is_none(), + "leaf ran when it should have been skipped (got {result:?})" + ); + + Ok(()) +} + +// Imported flow with its own flow_env contains a nested BranchOne whose +// inner step has a skip_if predicate. The branch sub-flow inside the +// imported flow has `root_job` pointing to the **top parent**, but its +// `flow_innermost_root_job` points to the imported flow — so the lookup +// must walk via flow_innermost_root_job to find the imported flow's scope, +// not jump straight to root_job (which would surface the parent's env and +// give the wrong answer). +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_env_imported_flow_with_nested_branch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + // Imported flow: own flow_env={KEY: "imported"}, contains a BranchOne + // whose inner step has skip_if = "flow_env.KEY === 'imported'". The + // predicate must see the IMPORTED flow's env, not the parent's + // ({KEY: "parent"}). + let imported_path = "f/system/imported_with_nested_branch"; + let imported_value = json!({ + "modules": [{ + "id": "router", + "value": { + "type": "branchone", + "branches": [{ + "summary": null, + "expr": "true", + "modules": [ + { + "id": "marker", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return {marker: \"from-imported-branch\"}; }", + "input_transforms": {} + } + }, + { + "id": "leaf", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return {ran: true}; }", + "input_transforms": {} + }, + "skip_if": { "expr": "flow_env.KEY === 'imported'" } + } + ], + "skip_failure": false, + "parallel": false, + }], + "default": [], + } + }], + "flow_env": { "KEY": "imported" } + }); + let imported_version_id: i64 = 9991003; + sqlx::query!( + "INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ($1, '', '', $2, ARRAY[$3]::bigint[], '{}'::jsonb, $4, 'system')", + "test-workspace", + imported_path, + imported_version_id, + imported_value.clone(), + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) VALUES ($1, $2, $3, '{}'::jsonb, $4, 'system')", + imported_version_id, + "test-workspace", + imported_path, + imported_value.clone(), + ) + .execute(&db) + .await?; + + let mut parent_env = std::collections::HashMap::new(); + parent_env.insert( + "KEY".to_string(), + windmill_common::worker::to_raw_value(&json!("parent")), + ); + + let parent = FlowValue { + modules: vec![flow_module( + "import", + FlowModuleValue::Flow { + input_transforms: Default::default(), + path: imported_path.to_string(), + pass_flow_input_directly: None, + }, + )], + flow_env: Some(parent_env), + same_worker: false, + ..Default::default() + }; + + let result = + RunJob::from(JobPayload::RawFlow { value: parent, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); + + // skip_if must see imported's env (KEY="imported") → predicate is true → + // leaf is skipped → branch returns marker's output. If the lookup + // shortcuts via root_job to the top parent, KEY would be "parent", + // skip_if would be false, leaf would run and return {ran: true}. + assert_eq!( + result["marker"], "from-imported-branch", + "skip_if inside imported flow's branch must see imported's flow_env, not parent's (got {result:?})" + ); + assert!( + result.get("ran").is_none(), + "leaf ran — predicate didn't see imported flow's flow_env scope (got {result:?})" + ); + + Ok(()) +} + +// stop_after_if predicate sees flow_env. Regression for the eval at line 614 +// of `update_flow_status_after_job_completion_internal` which used to pass +// `None` for flow_env unconditionally. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_env_in_stop_after_if(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + let mut flow_env = std::collections::HashMap::new(); + flow_env.insert( + "STOP".to_string(), + windmill_common::worker::to_raw_value(&json!(true)), + ); + + let first = { + let mut m = flow_module( + "first", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" +export function main() { + return {stage: "first"}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + m.stop_after_if = Some(windmill_common::flows::StopAfterIf { + expr: "flow_env.STOP === true".to_string(), + skip_if_stopped: true, + error_message: None, + }); + m + }; + + let second = flow_module( + "second", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" +export function main() { + return {stage: "second"}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + + let flow = FlowValue { + modules: vec![first, second], + flow_env: Some(flow_env), + same_worker: false, + ..Default::default() + }; + + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); + + // With fix: stop_after_if reads flow_env.STOP=true → flow stops early + // after `first`, result is first's output. + // Without fix: stop_after_if sees flow_env=None, predicate is false, the + // flow continues to `second` whose output overrides the result. + assert_eq!( + result["stage"], "first", + "stop_after_if with flow_env should stop after `first`; got {result:?}" + ); + + Ok(()) +} + +// retry_if predicate sees flow_env. Regression for the two evaluate_retry +// call sites in `update_flow_status_after_job_completion_internal` (lines +// 1194 and 1576) which used to pass `None` for flow_env. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_env_in_retry_if(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + let mut flow_env = std::collections::HashMap::new(); + flow_env.insert( + "SHOULD_RETRY".to_string(), + windmill_common::worker::to_raw_value(&json!(true)), + ); + + let fails = { + let mut m = flow_module( + "fails", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" +export function main() { + throw new Error("nope"); +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + m.retry = Some(windmill_common::flows::Retry { + constant: windmill_common::flows::ConstantDelay { attempts: 2, seconds: 0 }, + exponential: Default::default(), + retry_if: Some(windmill_common::flows::RetryIf { + expr: "flow_env.SHOULD_RETRY === true".to_string(), + }), + }); + m + }; + + let flow = FlowValue { + modules: vec![fails], + flow_env: Some(flow_env), + same_worker: false, + ..Default::default() + }; + + let completed = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await; + + // The flow always fails (script throws every attempt), but retry_if + // controls whether retries happen at all. With the fix, retry_if sees + // flow_env.SHOULD_RETRY=true and retries fire (fail_count > 0). Without + // the fix, the predicate gets `None` for flow_env, evaluates to false, + // and the flow fails on the first attempt with fail_count = 0. + let flow_status = completed + .flow_status + .as_ref() + .expect("flow should have a flow_status"); + let module_status = &flow_status["modules"][0]; + let failed_retries = module_status["failed_retries"].as_array(); + assert!( + failed_retries.is_some_and(|v| !v.is_empty()), + "retry_if with flow_env should have triggered retries; module status: {module_status:?}" + ); + + Ok(()) +} + +// stop_after_all_iters_if predicate sees flow_env. Regression for the +// signature change to `evaluate_stop_after_all_iters_if`. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_env_in_stop_after_all_iters_if(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + let mut flow_env = std::collections::HashMap::new(); + flow_env.insert( + "STOP".to_string(), + windmill_common::worker::to_raw_value(&json!(true)), + ); + + let inner = flow_module( + "iter_step", + FlowModuleValue::RawScript { + input_transforms: [js_input("i", "flow_input.iter.value")].into(), + language: ScriptLang::Deno, + content: r#" +export function main(i: number) { + return {iter: i}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + + let loop_module = { + let mut m = flow_module( + "loop", + FlowModuleValue::ForloopFlow { + iterator: InputTransform::Javascript { expr: "[1, 2, 3]".to_string() }, + modules: vec![inner], + modules_node: None, + skip_failures: false, + parallel: false, + parallelism: None, + squash: None, + }, + ); + m.stop_after_all_iters_if = Some(windmill_common::flows::StopAfterIf { + expr: "flow_env.STOP === true".to_string(), + skip_if_stopped: true, + error_message: None, + }); + m + }; + + let after = flow_module( + "after", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" +export function main() { + return {stage: "after-loop"}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); + + let flow = FlowValue { + modules: vec![loop_module, after], + flow_env: Some(flow_env), + same_worker: false, + ..Default::default() + }; + + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); + + // With fix: stop_after_all_iters_if reads flow_env.STOP=true after the + // loop completes → flow stops, `after` does not run, final result is + // the loop's output. + // Without fix: predicate sees flow_env=None, returns false, `after` runs + // and overrides the result. + assert!( + result.get("stage").is_none() || result["stage"] != "after-loop", + "stop_after_all_iters_if with flow_env should stop after the loop; got {result:?}" + ); + + Ok(()) +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 9907e35529..1ed5c7c9e4 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -343,10 +343,6 @@ pub fn workspaced_service() -> Router { "/result_by_id/{job_id}/{node_id}", get(get_result_by_id).layer(cors.clone()), ) - .route( - "/flow_env_by_flow_job_id/{flow_job_id}/{var_name}", - get(get_flow_env_by_flow_job_id).layer(cors.clone()), - ) .route("/run/dependencies", post(run_dependencies_job)) .route("/run/dependencies_async", post(run_dependencies_job_async)) .route("/run/flow_dependencies", post(run_flow_dependencies_job)) @@ -452,130 +448,6 @@ async fn get_root_job( Ok(Json(res)) } -async fn get_flow_env_by_flow_job_id( - authed: ApiAuthed, - tokened: Tokened, - Extension(db): Extension, - Path((w_id, flow_job_id, var_name)): Path<(String, Uuid, String)>, - Query(JsonPath { json_path, .. }): Query, -) -> windmill_common::error::JsonResult> { - // Fetch raw value (without json_path) to check for $var:/$res: references - let raw_value = sqlx::query_scalar!( - r#" - SELECT - CASE - WHEN flow_version.id IS NOT NULL THEN - flow_version.value -> 'flow_env' -> $3 - ELSE - root_job.raw_flow -> 'flow_env' -> $3 - END AS "flow_env: sqlx::types::Json>" - FROM - v2_job current_job - JOIN - v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id) - AND root_job.workspace_id = current_job.workspace_id - LEFT JOIN - flow_version ON flow_version.id = root_job.runnable_id - AND flow_version.path = root_job.runnable_path - AND flow_version.workspace_id = root_job.workspace_id - WHERE - current_job.id = $1 AND - current_job.workspace_id = $2"#, - flow_job_id, - w_id, - var_name, - ) - .fetch_optional(&db) - .await? - .and_then(|r| r.map(|x| x.0)); - - // Resolve $var:/$res: references if present - let resolved = if let Some(raw) = raw_value { - let raw_str = raw.get(); - let db_authed = windmill_common::db::DbWithOptAuthed::::from_authed( - &authed, - db.clone(), - None, - ); - if let Some(path) = raw_str - .strip_prefix("\"$var:") - .and_then(|s| s.strip_suffix("\"")) - { - match windmill_store::variables::get_value_internal(&db_authed, &w_id, path, false) - .await - { - Ok(val) => to_raw_value(&serde_json::Value::String(val)), - Err(e) => { - tracing::warn!("Failed to resolve flow_env variable $var:{path}: {e}"); - raw - } - } - } else if let Some(path) = raw_str - .strip_prefix("\"$res:") - .and_then(|s| s.strip_suffix("\"")) - { - match windmill_store::resources::get_resource_value_interpolated_internal( - &db_authed, - &w_id, - path, - Some(flow_job_id), - Some(&tokened.token), - false, - ) - .await - { - Ok(Some(val)) => to_raw_value(&val), - Ok(None) => { - tracing::warn!( - "Failed to resolve flow_env resource $res:{path}: resource not found" - ); - raw - } - Err(e) => { - tracing::warn!("Failed to resolve flow_env resource $res:{path}: {e}"); - raw - } - } - } else { - raw - } - } else { - to_raw_value(&serde_json::Value::Null) - }; - - // Apply json_path navigation on the (possibly resolved) value - let flow_env = if let Some(ref jp) = json_path { - let mut value: serde_json::Value = - serde_json::from_str(resolved.get()).unwrap_or(serde_json::Value::Null); - for part in jp.split('.') { - value = match value { - serde_json::Value::Object(ref mut map) => { - map.remove(part).unwrap_or(serde_json::Value::Null) - } - serde_json::Value::Array(ref arr) => part - .parse::() - .ok() - .and_then(|i| arr.get(i).cloned()) - .unwrap_or(serde_json::Value::Null), - _ => serde_json::Value::Null, - }; - } - to_raw_value(&value) - } else { - resolved - }; - - log_job_view( - &db, - Some(&authed), - Some(&tokened.token), - &w_id, - &flow_job_id, - ) - .await?; - Ok(Json(flow_env)) -} - async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result { let root_job = sqlx::query_scalar!( r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, diff --git a/backend/windmill-common/src/client.rs b/backend/windmill-common/src/client.rs index 730136ac1c..68670cea31 100644 --- a/backend/windmill-common/src/client.rs +++ b/backend/windmill-common/src/client.rs @@ -134,26 +134,6 @@ impl AuthedClient { .await } - pub async fn get_flow_env_by_flow_job_id( - &self, - root_job_id: &str, - var_name: &str, - json_path: Option, - ) -> anyhow::Result { - let url = format!( - "{}/api/w/{}/jobs/flow_env_by_flow_job_id/{}/{}", - self.base_internal_url, self.workspace, root_job_id, var_name - ); - let query = query_from_json_path(json_path); - make_basic_get_request( - self, - &url, - Some(query), - Some("decoding flow env variable as json"), - ) - .await - } - pub async fn get_result_by_id( &self, flow_job_id: &str, diff --git a/backend/windmill-jseval/src/lib.rs b/backend/windmill-jseval/src/lib.rs index 7172e8cfc8..f522f95263 100644 --- a/backend/windmill-jseval/src/lib.rs +++ b/backend/windmill-jseval/src/lib.rs @@ -56,12 +56,21 @@ const END_BRACKET_PATTERN: &str = "\"]"; // ── Regex statics ───────────────────────────────────────────────────── lazy_static! { + // `results` is fetched lazily via the `__getResult` async proxy, so we + // wrap each `results.X` access with `(await ...)` to drive the proxy. + // `flow_env` used to be wrapped here too (it was an async Deno op-backed + // proxy in the deno_core era); QuickJS now exposes flow_env as a plain + // in-memory object, so no await is needed. static ref RE: Regex = Regex::new( - r#"(?m)(?P(?:results|flow_env)(?:\?)?(?:(?:\.[a-zA-Z_0-9]+)|(?:\[\".*?\"\])))"# + r#"(?m)(?Presults(?:\?)?(?:(?:\.[a-zA-Z_0-9]+)|(?:\[\".*?\"\])))"# ) .unwrap(); + // SQL fast-path: simple `results.X.Y[i]...` accesses are dispatched to + // the API endpoint to fetch a specific result without spinning the eval + // engine. flow_env is no longer dispatched here because QuickJS reads + // it directly from the in-process global set up by `eval_quickjs_inner`. static ref RE_FULL: Regex = Regex::new( - r"(?m)^(results|flow_env)(?:\?)?\.([a-zA-Z_0-9]+)(?:\[(\d+)\])?((?:\.[a-zA-Z_0-9]+)+)?$" + r"(?m)^results(?:\?)?\.([a-zA-Z_0-9]+)(?:\[(\d+)\])?((?:\.[a-zA-Z_0-9]+)+)?$" ) .unwrap(); } @@ -173,10 +182,9 @@ pub async fn handle_full_regex( by_id: &IdContext, ) -> Option>> { if let Some(captures) = RE_FULL.captures(&expr) { - let obj_name = captures.get(1).unwrap().as_str(); - let obj_key = captures.get(2).unwrap().as_str(); - let idx_o = captures.get(3).map(|y| y.as_str()); - let rest = captures.get(4).map(|y| y.as_str()); + let obj_key = captures.get(1).unwrap().as_str(); + let idx_o = captures.get(2).map(|y| y.as_str()); + let rest = captures.get(3).map(|y| y.as_str()); // Skip the SQL fast path when the expression accesses a JS runtime // property (e.g. .length) that the PostgreSQL #> operator can't resolve. @@ -193,33 +201,19 @@ pub async fn handle_full_regex( rest.map(|x| x.trim_start_matches('.').to_string()) }; - let result = if obj_name == "results" { - let res = authed_client - .get_result_by_id::>>( - &by_id.flow_job.to_string(), - obj_key, - query, - ) - .await - .ok() - .flatten(); - match res { - Some(v) => Ok(v), - None => serde_json::value::to_raw_value(&serde_json::Value::Null) - .map_err(|e| anyhow::anyhow!("Failed to serialize null: {}", e)), - } - } else if obj_name == "flow_env" { - authed_client - .get_flow_env_by_flow_job_id(&by_id.flow_job.to_string(), obj_key, query) - .await - } else { - unreachable!(); + let res = authed_client + .get_result_by_id::>>(&by_id.flow_job.to_string(), obj_key, query) + .await + .ok() + .flatten(); + let result = match res { + Some(v) => Ok(v), + None => serde_json::value::to_raw_value(&serde_json::Value::Null) + .map_err(|e| anyhow::anyhow!("Failed to serialize null: {}", e)), }; - return Some(result); } - - return None; + None } #[cfg(feature = "quickjs")] diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index c21f22045a..eaa4fb8b6d 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -333,6 +333,7 @@ async fn evaluate_stop_after_all_iters_if( stop_early_err_msg: &mut Option, nresult: &mut Option>>, args: HashMap>, + flow_env: Option<&HashMap>>, flow: uuid::Uuid, status: &FlowStatus, ) -> error::Result<()> { @@ -354,7 +355,7 @@ async fn evaluate_stop_after_all_iters_if( let stop_early_after_all_iters = compute_bool_from_expr( &stop_after_all_iters_if.expr, Marc::new(args), - None, + flow_env, iters_result.clone(), None, id_ctx.as_ref(), @@ -465,6 +466,26 @@ pub async fn update_flow_status_after_job_completion_internal( has_triggered_error_handler = false; } + // Resolve flow_env for predicate evaluations (stop_after_if, + // stop_after_all_iters_if, retry_if). Only fetch when one of these + // predicates is configured to avoid an extra DB query on the common + // path. `retry` without `retry_if` doesn't consult flow_env. + let retry_uses_flow_env = + |module: &FlowModule| module.retry.as_ref().is_some_and(|r| r.retry_if.is_some()); + let needs_flow_env = current_module.is_some_and(|m| { + m.stop_after_if.is_some() + || m.stop_after_all_iters_if.is_some() + || retry_uses_flow_env(m) + }) || flow_value + .failure_module + .as_ref() + .is_some_and(|fm| retry_uses_flow_env(fm)); + let resolved_flow_env: Option>> = if needs_flow_env { + resolve_flow_env_for_status_update(db, client, flow, w_id, flow_value).await + } else { + None + }; + let module_status = match module_step { Step::PreprocessorStep => old_status .preprocessor_module @@ -611,7 +632,7 @@ pub async fn update_flow_status_after_job_completion_internal( let bool_res = compute_bool_from_expr( &expr, Marc::new(args), - None, + resolved_flow_env.as_ref(), result.clone(), all_iters, id_ctx.as_ref(), @@ -895,6 +916,7 @@ pub async fn update_flow_status_after_job_completion_internal( &mut stop_early_err_msg, &mut nresult, args, + resolved_flow_env.as_ref(), flow, &old_status, ) @@ -1114,6 +1136,7 @@ pub async fn update_flow_status_after_job_completion_internal( &mut stop_early_err_msg, &mut nresult, args, + resolved_flow_env.as_ref(), flow, &old_status, ) @@ -1196,7 +1219,7 @@ pub async fn update_flow_status_after_job_completion_internal( &old_status.retry, result.clone(), Marc::new(args), - None, + resolved_flow_env.as_ref(), Some(client), ) .await? @@ -1578,7 +1601,7 @@ pub async fn update_flow_status_after_job_completion_internal( &old_status.retry, result.clone(), Marc::new(args), - None, + resolved_flow_env.as_ref(), Some(client), ) .await? @@ -2358,6 +2381,116 @@ async fn fetch_root_flow_id(db: &DB, flow_id: Uuid) -> Uuid { .unwrap_or(flow_id) } +// Resolve the flow_env to use for predicate evaluations in +// `update_flow_status_after_job_completion_internal`: take the current flow's +// `flow_env` if present, otherwise inherit from the root flow, then interpolate +// any `$var:`/`$res:` references via `transform_json`. +async fn resolve_flow_env_for_status_update( + db: &DB, + client: &AuthedClient, + flow_job_id: Uuid, + workspace_id: &str, + flow_value: &FlowValue, +) -> Option>> { + let env = if let Some(ref e) = flow_value.flow_env { + e.clone() + } else { + fetch_root_flow_env(db, flow_job_id, workspace_id).await? + }; + if env.is_empty() { + return Some(env); + } + let mini = match get_mini_pulled_job(db, &flow_job_id).await { + Ok(Some(j)) => j, + Ok(None) => return Some(env), + Err(e) => { + tracing::warn!("Failed to fetch flow job to resolve flow_env: {e:#}"); + return Some(env); + } + }; + match transform_json( + client, + workspace_id, + &env, + &mini, + &Connection::Sql(db.clone()), + ) + .await + { + Ok(Some(resolved)) => Some(resolved), + Ok(None) => Some(env), + Err(e) => { + tracing::warn!("Failed to resolve flow_env references in status update: {e:#}"); + Some(env) + } + } +} + +// Maximum number of ancestor jobs the flow_env walk will follow before bailing. +// Bounds runtime cost and protects against pathological data. +const MAX_FLOW_ENV_LOOKUP_DEPTH: i32 = 50; + +// Look up the nearest ancestor flow's `flow_env` for the given flow job. +// Sub-flows spawned by branches/loops via `payload_from_modules` don't carry +// `flow_env` in their own `FlowValue`, so any predicate evaluated against the +// local flow_env would see `None`. We walk the ancestor chain via +// `flow_innermost_root_job → parent_job` (one step at a time) and return the +// CLOSEST ancestor whose persisted flow definition (`flow_version.value` or +// `raw_flow`) carries `flow_env`. +// +// The walk deliberately does NOT use `root_job` (which would jump straight to +// the topmost parent) because imported flows define their own `flow_env` +// scope: a branch inside an imported flow must see the imported flow's env, +// not the top parent's. `flow_innermost_root_job` walks one scope at a time +// and resets to NULL on parallel-loop iterations and imported sub-flows, so +// recursion is required to walk past those resets to the closest scope. +async fn fetch_root_flow_env( + db: &DB, + flow_job_id: Uuid, + workspace_id: &str, +) -> Option>> { + sqlx::query_scalar!( + r#"WITH RECURSIVE chain(id, parent_job, flow_innermost_root_job, runnable_id, runnable_path, raw_flow, depth) AS ( + SELECT id, parent_job, flow_innermost_root_job, runnable_id, runnable_path, raw_flow, 0 + FROM v2_job + WHERE id = $1 AND workspace_id = $2 + UNION ALL + SELECT j.id, j.parent_job, j.flow_innermost_root_job, j.runnable_id, j.runnable_path, j.raw_flow, c.depth + 1 + FROM v2_job j + JOIN chain c + ON j.id = COALESCE(c.flow_innermost_root_job, c.parent_job) + WHERE j.workspace_id = $2 AND c.depth < $3 + ) + SELECT + CASE + WHEN flow_version.id IS NOT NULL THEN + flow_version.value -> 'flow_env' + ELSE + chain.raw_flow -> 'flow_env' + END AS "flow_env: Json>>" + FROM chain + LEFT JOIN flow_version + ON flow_version.id = chain.runnable_id + AND flow_version.path = chain.runnable_path + AND flow_version.workspace_id = $2 + WHERE (CASE + WHEN flow_version.id IS NOT NULL THEN flow_version.value -> 'flow_env' + ELSE chain.raw_flow -> 'flow_env' + END) IS NOT NULL + ORDER BY chain.depth ASC + LIMIT 1"#, + flow_job_id, + workspace_id, + MAX_FLOW_ENV_LOOKUP_DEPTH, + ) + .fetch_optional(db) + .await + .ok() + .flatten() + .flatten() + .map(|json| json.0) +} + struct FailureContext { started_at: Arc>, flow_job_id: Uuid, @@ -2497,11 +2630,24 @@ pub async fn handle_flow( ) -> anyhow::Result<()> { let flow = flow_data.value(); + // Sub-flows spawned by `payload_from_modules` for branches/loops don't + // carry the parent's `flow_env` in their own FlowValue. Fall back to the + // nearest enclosing scope's `flow_env` so predicates like `skip_if`, + // `stop_after_if`, and branch conditions see the same env as input + // transforms. + let inherited_env: Option>> = + if flow.flow_env.is_none() && flow_job.parent_job.is_some() { + fetch_root_flow_env(db, flow_job.id, &flow_job.workspace_id).await + } else { + None + }; + let env_source = flow.flow_env.as_ref().or(inherited_env.as_ref()); + // Resolve $var: and $res: references in flow_env. // We resolve into a separate variable to avoid cloning the entire FlowValue // (which includes modules, failure_module, etc.) just to replace flow_env. let resolved_env; - let flow_env = if let Some(ref env) = flow.flow_env { + let flow_env = if let Some(env) = env_source { match transform_json( client, &flow_job.workspace_id, @@ -2515,10 +2661,10 @@ pub async fn handle_flow( resolved_env = resolved; Some(&resolved_env) } - Ok(None) => flow.flow_env.as_ref(), + Ok(None) => Some(env), Err(e) => { tracing::warn!("Failed to resolve flow_env references: {e}"); - flow.flow_env.as_ref() + Some(env) } } } else { From 1bf1477cf7e534d7bf40c06e8b714212b965e6b4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 5 May 2026 20:25:41 +0000 Subject: [PATCH 31/56] ci: run codex review on every follow-up commit --- .github/workflows/codex-pr-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index eacb453cc3..e89a58b629 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -2,7 +2,7 @@ name: Codex Auto Review on: pull_request: - types: [ready_for_review, opened] + types: [ready_for_review, opened, synchronize] workflow_call: inputs: pr_number: From 40dbab531e5166b894f3f94b0d72b2ac456c0097 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 5 May 2026 21:06:53 +0000 Subject: [PATCH 32/56] fix(cli): resolve cross-folder relative imports during lockgen on fresh DB (#9048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(cli): resolve cross-folder relative imports during lockgen on fresh DB On a fresh workspace, lockfile generation for scripts that imported other scripts via cross-folder relative imports (or barrel re-exporters) failed with "Failed to find relative import" because the dep job's bun build hit the server before any helper was deployed. Three independent bugs combined to produce this: 1. wmill sync push --auto-metadata regenerated locks per script without building a DoubleLinkedDependencyTree or calling uploadScripts, so temp_script_refs was never sent to dependencies_async. 2. wmill script generate-metadata (the deprecated alias) had its own old in-line implementation that bypassed the tree entirely. 3. The TypeScript WASM parser dropped re-exports (export * from, export { x } from) when called with skip_type_only=false — the path used by parse_relative_imports — so barrel files looked like leaves to the CLI's dependency tree and their sibling helpers were missing from temp_script_refs. Fix: - sync.ts: --auto-metadata mirrors generate-metadata's flow (dryRun pass to populate tree → propagateStaleness → uploadScripts → real pass with tree). - script.ts: deprecated wmill script generate-metadata now delegates to the canonical generateMetadata, which already does the tree+upload dance. - parser-ts: visit_export_all and visit_named_export had inverted skip_type_only guards; aligned with visit_import_decl's pattern. Includes 4 E2E tests reproducing each customer-hit failure path and a Rust unit test for the re-export parser fix. Co-Authored-By: Claude Opus 4.7 (1M context) * chore: bump windmill-parser-wasm-ts to 1.695.0 Pin the parser package to the version published with the re-export fix (visit_export_all / visit_named_export skip_type_only=false) so the CLI and frontend pick it up at the next release. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(cli): restore legacy stale-check in deprecated alias, add tree to gen pass Delegating wmill script generate-metadata fully to the canonical handler broke 4 workspace_deps_filter tests that rely on the legacy hash-with-deps formula and the "No metadata to update" output string. Restore the original in-line implementation (legacy stale-check preserved), but add a DoubleLinkedDependencyTree + uploadScripts pass before the actual generation step. The customer's bug only manifests on real lockgen, not on the dry-run staleness check, so this preserves the existing test contract while still fixing cross-folder relative imports for the deprecated alias. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- backend/parsers/windmill-parser-ts/src/lib.rs | 36 +-- .../parsers/windmill-parser-ts/tests/tests.rs | 23 ++ cli/bun.lock | 4 +- cli/package.json | 2 +- .../generate-metadata/generate-metadata.ts | 2 +- cli/src/commands/script/script.ts | 40 ++- cli/src/commands/sync/sync.ts | 187 +++++++++--- .../sync_push_auto_metadata_repro.test.ts | 270 ++++++++++++++++++ frontend/package-lock.json | 8 +- frontend/package.json | 2 +- 10 files changed, 501 insertions(+), 73 deletions(-) create mode 100644 cli/test/sync_push_auto_metadata_repro.test.ts diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index e63b0ef680..1ae22879b6 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -77,32 +77,36 @@ impl Visit for ImportsFinder { } fn visit_export_all(&mut self, node: &swc_ecma_ast::ExportAll) { - if !self.skip_type_only || node.type_only { + if self.skip_type_only && node.type_only { return; } self.process_raw(node.src.raw.as_ref().map(|x| x.to_string())); } fn visit_named_export(&mut self, node: &swc_ecma_ast::NamedExport) { - if node.src.is_none() || !self.skip_type_only || node.type_only { + if node.src.is_none() { return; } - if node.specifiers.len() > 0 { - let mut is_type_only = true; - for specifier in node.specifiers.iter() { - match specifier { - swc_ecma_ast::ExportSpecifier::Named(swc_ecma_ast::ExportNamedSpecifier { - is_type_only, - .. - }) if *is_type_only => (), - _ => { - is_type_only = false; - break; + if self.skip_type_only { + if node.type_only { + return; + } + if node.specifiers.len() > 0 { + let mut is_type_only = true; + for specifier in node.specifiers.iter() { + match specifier { + swc_ecma_ast::ExportSpecifier::Named( + swc_ecma_ast::ExportNamedSpecifier { is_type_only, .. }, + ) if *is_type_only => (), + _ => { + is_type_only = false; + break; + } } } - } - if is_type_only { - return; + if is_type_only { + return; + } } } diff --git a/backend/parsers/windmill-parser-ts/tests/tests.rs b/backend/parsers/windmill-parser-ts/tests/tests.rs index 0243ccf06d..4309018fb4 100644 --- a/backend/parsers/windmill-parser-ts/tests/tests.rs +++ b/backend/parsers/windmill-parser-ts/tests/tests.rs @@ -936,4 +936,27 @@ mod tests { let result = parse_relative_imports(code, "f/one/two/three/script").unwrap(); assert_eq!(result, vec!["f/b", "f/one/a"]); } + + #[test] + fn test_relative_imports_includes_re_exports() { + // Barrel re-exports must be captured as relative imports — without + // them, importers reaching helpers via a barrel file lose the edge in + // the dependency tree and the dep job 404s on the sibling fetches. + let code = r#" + export * from "./types.ts"; + export { WorkflowError } from "./WorkflowError.ts"; + export * as factory from "./errorFactory.ts"; + export type { ErrorKind } from "./types-only.ts"; + "#; + let result = parse_relative_imports(code, "f/lib/errors/index").unwrap(); + assert_eq!( + result, + vec![ + "f/lib/errors/WorkflowError", + "f/lib/errors/errorFactory", + "f/lib/errors/types", + "f/lib/errors/types-only", + ] + ); + } } diff --git a/cli/bun.lock b/cli/bun.lock index f4cb387593..de02781822 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -28,7 +28,7 @@ "windmill-parser-wasm-regex": "1.692.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", - "windmill-parser-wasm-ts": "1.693.1", + "windmill-parser-wasm-ts": "1.695.0", "windmill-parser-wasm-yaml": "1.593.0", "windmill-yaml-validator": "1.1.1", "ws": "8.18.0", @@ -305,7 +305,7 @@ "windmill-parser-wasm-rust": ["windmill-parser-wasm-rust@1.647.1", "", {}, "sha512-9yGLYZX2Hn9TdTqGY/5Fp50ftzgUsrfBkSK9vJkKJd5Amyg+yXLBGzd8pz6Org+4uxMenz/16wpsgijvo6uhhQ=="], - "windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.693.1", "", {}, "sha512-xrPgVWwQbOWJKiz68wBDNMrKVOtCY/utyhzSx0kFYIWm/QH/6L8k6LLjo4DOn+PnlBNRXc7qum0sDyB8IuuJYQ=="], + "windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.695.0", "", {}, "sha512-9EFxeRZWmfb7EyhSlcG7dzTTKETPRYAvpRlxxLkhhtI5I219wFgI7kwrMpz4stXHJj/aqBknVv66NQHRstSJmw=="], "windmill-parser-wasm-yaml": ["windmill-parser-wasm-yaml@1.593.0", "", {}, "sha512-Gyx4aR2jsJYuDrD3mCNTmz7LWOQQXPw5yKNCC1xRgUOPfjsD/tINAFfsBLwVOSmlQQcFZO+wHm4KtDtXOcnGVw=="], diff --git a/cli/package.json b/cli/package.json index 08c3d0c693..5afe6f04bb 100644 --- a/cli/package.json +++ b/cli/package.json @@ -37,7 +37,7 @@ "windmill-parser-wasm-regex": "1.692.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", - "windmill-parser-wasm-ts": "1.693.1", + "windmill-parser-wasm-ts": "1.695.0", "windmill-parser-wasm-yaml": "1.593.0", "windmill-yaml-validator": "1.1.1", "ws": "8.18.0", diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index e840397230..226220592e 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -263,7 +263,7 @@ export async function rehashOnly( return counts; } -async function generateMetadata( +export async function generateMetadata( opts: GlobalOptions & { yes?: boolean; lockOnly?: boolean; diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index db2b9b3fef..0635cd42ea 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -1380,7 +1380,42 @@ export async function generateMetadata( log.info(colors.green.bold("No metadata to update")); return; } - // TODO: test this + + // Build a DoubleLinkedDependencyTree and upload mismatched scripts to + // raw_script_temp before the actual generation pass. Without this, + // dep jobs for scripts that import other not-yet-deployed scripts via + // relative paths would 404 on the import target (the very bug this + // alias was introducing on fresh-DB pushes). + const { DoubleLinkedDependencyTree, uploadScripts } = await import( + "../../utils/dependency_tree.ts" + ); + const tree = new DoubleLinkedDependencyTree(); + tree.setWorkspaceDeps(rawWorkspaceDependencies); + for (const e of Object.keys(elems)) { + await generateScriptMetadataInternal( + e, + workspace, + opts, + true, // dryRun: populate tree + true, + rawWorkspaceDependencies, + codebases, + false, + tree, + ); + } + tree.propagateStaleness(); + try { + await uploadScripts(tree, workspace); + } catch (e) { + log.warn( + colors.yellow( + `Failed to upload scripts to temp storage (backend may be too old): ${e}. ` + + `Locks will be generated using deployed script versions only — locally modified ` + + `relative imports may not be reflected.`, + ), + ); + } for (const e of Object.keys(elems)) { await generateScriptMetadataInternal( e, @@ -1390,7 +1425,8 @@ export async function generateMetadata( true, rawWorkspaceDependencies, codebases, - false + false, + tree, ); } } diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index a7dd91b5fa..0cadc7c083 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -79,6 +79,7 @@ import { MalformedLockfileError, workspaceDependenciesPathToLanguageAndFilename, } from "../../utils/metadata.ts"; +import { DoubleLinkedDependencyTree, uploadScripts } from "../../utils/dependency_tree.ts"; import { OpenFlow, NativeServiceName, ScriptModule } from "../../../gen/types.gen.ts"; import { pushResource } from "../resource/resource.ts"; import { @@ -2990,21 +2991,159 @@ export async function push( const staleFlows: string[] = []; const staleApps: string[] = []; + // Auto-regenerate uses a DoubleLinkedDependencyTree so the dep job can + // resolve cross-folder relative imports against not-yet-deployed scripts via + // raw_script_temp + temp_script_refs. Without this the importer's lockgen + // 404s on its sibling/parent imports because nothing has been pushed yet. + const tree = autoRegenerate ? new DoubleLinkedDependencyTree() : undefined; + if (tree) tree.setWorkspaceDeps(rawWorkspaceDependencies); + + // Pass 1: populate the tree (autoRegenerate) or run the legacy stale-check + // (no autoRegenerate, just collect warnings). for (const change of tracker.scripts) { const stale = await generateScriptMetadataInternal( change, workspace, opts, - !autoRegenerate, // dryRun=false when --auto is set + true, // dryRun: pass 1 only populates the tree / detects staleness true, rawWorkspaceDependencies, codebases, false, + tree, ); - if (stale) { + if (!autoRegenerate && stale) { staleScripts.push(stale); } } + for (const change of tracker.flows) { + const stale = await generateFlowLockInternal( + change, + true, + workspace, + opts, + false, + true, + tree, + ); + if (!autoRegenerate && stale) { + staleFlows.push(stale as string); + } + } + for (const change of tracker.apps) { + const stale = await generateAppLocksInternal( + change, + false, + true, + workspace, + opts, + true, + true, + tree, + ); + if (!autoRegenerate && stale) { + staleApps.push(stale as string); + } + } + for (const change of tracker.rawApps) { + const stale = await generateAppLocksInternal( + change, + true, + true, + workspace, + opts, + true, + true, + tree, + ); + if (!autoRegenerate && stale) { + staleApps.push(stale as string); + } + } + + if (autoRegenerate && tree) { + // Propagate staleness through imports + upload script content to + // raw_script_temp so the dep job can resolve cross-folder relative imports + // via temp_script_refs (instead of hitting 404s for not-yet-deployed + // scripts and recording lock_error_logs). + tree.propagateStaleness(); + try { + await uploadScripts(tree, workspace); + } catch (e) { + log.warn( + colors.yellow( + `Failed to upload scripts to temp storage (backend may be too old): ${e}. ` + + `Locks will be generated using deployed script versions only — locally modified ` + + `relative imports may not be reflected.`, + ), + ); + } + + // Pass 2: actually generate metadata/locks. Threading `tree` makes + // generateScriptMetadataInternal include temp_script_refs in the + // dependencies_async request so the dep job resolves relative imports + // against raw_script_temp. + for (const change of tracker.scripts) { + const generated = await generateScriptMetadataInternal( + change, + workspace, + opts, + false, + true, + rawWorkspaceDependencies, + codebases, + false, + tree, + ); + if (generated) { + staleScripts.push(generated); + } + } + for (const change of tracker.flows) { + const generated = await generateFlowLockInternal( + change, + false, + workspace, + opts, + false, + true, + tree, + ); + if (generated) { + staleFlows.push(generated as string); + } + } + for (const change of tracker.apps) { + const generated = await generateAppLocksInternal( + change, + false, + false, + workspace, + opts, + true, + true, + tree, + ); + if (generated) { + staleApps.push(generated as string); + } + } + for (const change of tracker.rawApps) { + const generated = await generateAppLocksInternal( + change, + true, + false, + workspace, + opts, + true, + true, + tree, + ); + if (generated) { + staleApps.push(generated as string); + } + } + } if (staleScripts.length > 0) { log.info(""); @@ -3026,20 +3165,6 @@ export async function push( log.info(""); } - for (const change of tracker.flows) { - const stale = await generateFlowLockInternal( - change, - !autoRegenerate, // dryRun=false when --auto is set - workspace, - opts, - false, - true, - ); - if (stale) { - staleFlows.push(stale as string); - } - } - if (staleFlows.length > 0) { if (autoRegenerate) { log.info("Auto-regenerated locks for stale flows:"); @@ -3058,36 +3183,6 @@ export async function push( log.info(""); } - for (const change of tracker.apps) { - const stale = await generateAppLocksInternal( - change, - false, - !autoRegenerate, - workspace, - opts, - true, - true, - ); - if (stale) { - staleApps.push(stale as string); - } - } - - for (const change of tracker.rawApps) { - const stale = await generateAppLocksInternal( - change, - true, - !autoRegenerate, - workspace, - opts, - true, - true, - ); - if (stale) { - staleApps.push(stale as string); - } - } - if (staleApps.length > 0) { if (autoRegenerate) { log.info("Auto-regenerated locks for stale apps:"); diff --git a/cli/test/sync_push_auto_metadata_repro.test.ts b/cli/test/sync_push_auto_metadata_repro.test.ts new file mode 100644 index 0000000000..e9ea167ffc --- /dev/null +++ b/cli/test/sync_push_auto_metadata_repro.test.ts @@ -0,0 +1,270 @@ +/** + * Reproduction for `sync push --auto-metadata` cross-folder relative-import bug. + * + * `--auto-metadata` regenerates lockfiles client-side before pushing, but on a + * fresh workspace none of the imported scripts are deployed yet. The fix must + * route lockgen through `DoubleLinkedDependencyTree` + `uploadScripts` so the + * dep job can resolve relative imports via `temp_script_refs`. + * + * Without the fix, `wmill sync push --auto-metadata --yes` aborts with + * "Failed to find relative import" / "Non-zero exit status for bun build". + */ + +import { expect, test } from "bun:test"; +import { writeFile, readFile } from "node:fs/promises"; +import { withTestBackend } from "./test_backend.ts"; +import { createLocalScript } from "./test_fixtures.ts"; + +const wmillYaml = `defaultTs: bun +includes: ["**"] +excludes: [] +`; + +// The importer path (f/aaa/...) sorts before the imported path (f/bbb/...) +// alphabetically. The CLI sorts changes by path within the script bucket, so +// without the fix the importer is processed first and its lockgen tries to +// fetch a not-yet-uploaded helper from the server. +// +// One `../` from f/aaa/consumer.ts steps out of f/aaa/ to f/, then `bbb/helper.ts` +// resolves to f/bbb/helper.ts. +const consumerScript = `import { helper } from "../bbb/helper.ts"; +export async function main() { return helper(); } +`; + +const helperScript = `export function helper() { return "ok"; } +`; + +test( + "sync push --auto-metadata succeeds for cross-folder relative imports on a fresh workspace", + { timeout: 120000 }, + async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, wmillYaml); + + // Cross-folder relative import where the importer's path comes + // alphabetically before the import target. + await createLocalScript(tempDir, "f/aaa", "consumer", "bun", consumerScript); + await createLocalScript(tempDir, "f/bbb", "helper", "bun", helperScript); + + const result = await backend.runCLICommand( + ["sync", "push", "--yes", "--auto-metadata"], + tempDir, + ); + + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + + // The exit code must be zero — `--auto-metadata` should not abort on a + // fresh workspace just because the importer is alphabetically first. + expect(result.code).toBe(0); + + const combined = result.stdout + result.stderr; + expect(combined).not.toContain("Failed to find relative import"); + expect(combined).not.toContain("Failed to generate lockfile"); + + // Consumer's lockfile should exist and be non-empty (i.e. lockgen + // actually produced a valid lock, not a sentinel/error string). + const consumerLock = await readFile( + `${tempDir}/f/aaa/consumer.script.lock`, + "utf-8", + ).catch(() => ""); + expect(consumerLock.length).toBeGreaterThan(0); + }); + }, +); + +// Helper: build a multi-folder topology mimicking the customer's failure +// shape. Importers (analytics, webhooks) reach helpers in f/lib via deep +// cross-folder relative imports. +async function setupCustomerLikeTopology(tempDir: string) { + await writeFile(`${tempDir}/wmill.yaml`, wmillYaml); + await createLocalScript( + tempDir, + "f/lib", + "log_event", + "bun", + `export function logEvent(msg: string) { return msg; }\n`, + ); + await createLocalScript( + tempDir, + "f/lib", + "errors", + "bun", + `export class AppError extends Error {}\n`, + ); + await createLocalScript( + tempDir, + "f/integrations/snowflake", + "client", + "bun", + `export function client() { return "snowflake"; }\n`, + ); + await createLocalScript( + tempDir, + "f/analytics/claims_operations", + "bulk", + "bun", + `import { client } from "../../integrations/snowflake/client.ts"; +import { AppError } from "../../lib/errors.ts"; +export async function main() { try { return client(); } catch (e) { throw new AppError(); } } +`, + ); + await createLocalScript( + tempDir, + "f/webhooks/stripe", + "handle_webhook", + "bun", + `import { logEvent } from "../../lib/log_event.ts"; +export async function main() { return logEvent("ok"); } +`, + ); +} + +test( + "generate-metadata succeeds across many folders with deep cross-folder relative imports", + { timeout: 180000 }, + async () => { + await withTestBackend(async (backend, tempDir) => { + await setupCustomerLikeTopology(tempDir); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir, + ); + + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + + expect(result.code).toBe(0); + + const combined = result.stdout + result.stderr; + expect(combined).not.toContain("Failed to find relative import"); + expect(combined).not.toContain("Failed to generate lockfile"); + + const bulkLock = await readFile( + `${tempDir}/f/analytics/claims_operations/bulk.script.lock`, + "utf-8", + ).catch(() => ""); + const webhookLock = await readFile( + `${tempDir}/f/webhooks/stripe/handle_webhook.script.lock`, + "utf-8", + ).catch(() => ""); + expect(bulkLock.length).toBeGreaterThan(0); + expect(webhookLock.length).toBeGreaterThan(0); + }); + }, +); + +// `wmill script generate-metadata` is a deprecated alias defined in +// commands/script/script.ts. Its action handler used to be a separate +// implementation that didn't go through DoubleLinkedDependencyTree + +// uploadScripts, so on a fresh DB it hit the same out-of-order failure as +// `sync push --auto-metadata`. The fix delegates the alias to the canonical +// generateMetadata implementation. +// Customer scenario: a barrel file (f/lib/errors/index.ts) re-exports from +// siblings (./types.ts, ./WorkflowError.ts, ...). An importer in a different +// folder imports from the barrel. On a fresh DB, the dep job for the importer +// fetches index.ts via raw_unpinned + temp_script_hash, but bun's resolver +// then has to resolve the barrel's *sibling* imports — and those need to be +// in TEMP_SCRIPT_REFS too. +test( + "generate-metadata succeeds when importer reaches helpers via a barrel re-exporter", + { timeout: 180000 }, + async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, wmillYaml); + + await createLocalScript( + tempDir, + "f/lib/errors", + "types", + "bun", + `export type ErrorKind = "fatal" | "warn";\n` + + `export function _typesAnchor() { return null as unknown; }\n`, + ); + await createLocalScript( + tempDir, + "f/lib/errors", + "WorkflowError", + "bun", + `export class WorkflowError extends Error { kind = "fatal" as const; }\n` + + `export function _wfeAnchor() { return new WorkflowError(); }\n`, + ); + await createLocalScript( + tempDir, + "f/lib/errors", + "index", + "bun", + `export * from "./types.ts";\n` + + `export * from "./WorkflowError.ts";\n` + + `export function main() { return "barrel"; }\n`, + ); + await createLocalScript( + tempDir, + "f/analytics/claims_operations", + "bulk", + "bun", + `import { WorkflowError } from "../../lib/errors/index.ts"; +export async function main() { return new WorkflowError().message; } +`, + ); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir, + ); + + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + + expect(result.code).toBe(0); + + const combined = result.stdout + result.stderr; + expect(combined).not.toContain("Failed to find relative import"); + expect(combined).not.toContain("Failed to generate lockfile"); + }); + }, +); + +test( + "deprecated `wmill script generate-metadata` succeeds for cross-folder imports on a fresh workspace", + { timeout: 180000 }, + async () => { + await withTestBackend(async (backend, tempDir) => { + await setupCustomerLikeTopology(tempDir); + + const result = await backend.runCLICommand( + ["script", "generate-metadata", "--yes"], + tempDir, + ); + + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + + expect(result.code).toBe(0); + + const combined = result.stdout + result.stderr; + expect(combined).not.toContain("Failed to find relative import"); + expect(combined).not.toContain("Failed to generate lockfile"); + + const bulkLock = await readFile( + `${tempDir}/f/analytics/claims_operations/bulk.script.lock`, + "utf-8", + ).catch(() => ""); + const webhookLock = await readFile( + `${tempDir}/f/webhooks/stripe/handle_webhook.script.lock`, + "utf-8", + ).catch(() => ""); + expect(bulkLock.length).toBeGreaterThan(0); + expect(webhookLock.length).toBeGreaterThan(0); + }); + }, +); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6fcd7f34c0..76002aba4a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -86,7 +86,7 @@ "windmill-parser-wasm-regex": "1.692.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", - "windmill-parser-wasm-ts": "1.693.1", + "windmill-parser-wasm-ts": "1.695.0", "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", @@ -13640,9 +13640,9 @@ "integrity": "sha512-9yGLYZX2Hn9TdTqGY/5Fp50ftzgUsrfBkSK9vJkKJd5Amyg+yXLBGzd8pz6Org+4uxMenz/16wpsgijvo6uhhQ==" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.693.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.693.1.tgz", - "integrity": "sha512-xrPgVWwQbOWJKiz68wBDNMrKVOtCY/utyhzSx0kFYIWm/QH/6L8k6LLjo4DOn+PnlBNRXc7qum0sDyB8IuuJYQ==" + "version": "1.695.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.695.0.tgz", + "integrity": "sha512-9EFxeRZWmfb7EyhSlcG7dzTTKETPRYAvpRlxxLkhhtI5I219wFgI7kwrMpz4stXHJj/aqBknVv66NQHRstSJmw==" }, "node_modules/windmill-parser-wasm-wac": { "version": "1.668.6", diff --git a/frontend/package.json b/frontend/package.json index 508e3813f8..98ff10ae9d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -159,7 +159,7 @@ "windmill-parser-wasm-regex": "1.692.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", - "windmill-parser-wasm-ts": "1.693.1", + "windmill-parser-wasm-ts": "1.695.0", "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", From c1e52eab09794746bea7dc9adb94552641f87d5b Mon Sep 17 00:00:00 2001 From: Henri Courdent <122811744+hcourdent@users.noreply.github.com> Date: Tue, 5 May 2026 23:12:00 +0200 Subject: [PATCH 33/56] fix: navigate home arrows (#9024) * Navigate with arrows * Jumps to other side item + load 30 more * No workspace selector * Recommendations Claude check * Navigation horizontal * Same --------- Co-authored-by: Ruben Fiszel --- .../lib/components/common/table/AppRow.svelte | 5 +- .../components/common/table/FlowRow.svelte | 5 +- .../components/common/table/RawAppRow.svelte | 5 +- .../lib/components/common/table/Row.svelte | 15 +- .../components/common/table/ScriptRow.svelte | 5 +- frontend/src/lib/components/home/Item.svelte | 13 +- .../src/lib/components/home/ItemsList.svelte | 254 +++++++++++++++++- 7 files changed, 292 insertions(+), 10 deletions(-) diff --git a/frontend/src/lib/components/common/table/AppRow.svelte b/frontend/src/lib/components/common/table/AppRow.svelte index 1034eeac81..1c0c5b08d1 100644 --- a/frontend/src/lib/components/common/table/AppRow.svelte +++ b/frontend/src/lib/components/common/table/AppRow.svelte @@ -46,6 +46,7 @@ depth?: number menuOpen?: boolean showEditButton?: boolean + keyboardSelected?: boolean } let { @@ -57,7 +58,8 @@ deleteConfirmedCallback = $bindable(), depth = 0, menuOpen = $bindable(false), - showEditButton = $bindable(true) + showEditButton = $bindable(true), + keyboardSelected = false }: Props = $props() const dispatch = createEventDispatcher() @@ -86,6 +88,7 @@ workspaceId={app.workspace_id ?? $workspaceStore ?? ''} canFavorite={!app.draft_only} {depth} + {keyboardSelected} > {#snippet badges()} {#if app.execution_mode == 'anonymous'} diff --git a/frontend/src/lib/components/common/table/FlowRow.svelte b/frontend/src/lib/components/common/table/FlowRow.svelte index ef0ff76215..14ee1a3287 100644 --- a/frontend/src/lib/components/common/table/FlowRow.svelte +++ b/frontend/src/lib/components/common/table/FlowRow.svelte @@ -49,6 +49,7 @@ depth?: number menuOpen?: boolean showEditButton?: boolean + keyboardSelected?: boolean } let { @@ -61,7 +62,8 @@ errorHandlerMuted, depth = 0, menuOpen = $bindable(false), - showEditButton = $bindable(true) + showEditButton = $bindable(true), + keyboardSelected = false }: Props = $props() const dispatch = createEventDispatcher() @@ -112,6 +114,7 @@ {errorHandlerMuted} canFavorite={!flow.draft_only} {depth} + {keyboardSelected} > {#snippet badges()} {#if flow.archived} diff --git a/frontend/src/lib/components/common/table/RawAppRow.svelte b/frontend/src/lib/components/common/table/RawAppRow.svelte index f163630747..e759a1d9fe 100644 --- a/frontend/src/lib/components/common/table/RawAppRow.svelte +++ b/frontend/src/lib/components/common/table/RawAppRow.svelte @@ -18,6 +18,7 @@ deploymentDrawer: DeployWorkspaceDrawer depth?: number menuOpen?: boolean + keyboardSelected?: boolean } let { @@ -26,7 +27,8 @@ shareModal, deploymentDrawer, depth = 0, - menuOpen = $bindable(false) + menuOpen = $bindable(false), + keyboardSelected = false }: Props = $props() @@ -39,6 +41,7 @@ workspaceId={app.workspace_id ?? $workspaceStore ?? ''} canFavorite={true} {depth} + {keyboardSelected} > {#snippet badges()} diff --git a/frontend/src/lib/components/common/table/Row.svelte b/frontend/src/lib/components/common/table/Row.svelte index f3d44965e4..58e3ed9a3f 100644 --- a/frontend/src/lib/components/common/table/Row.svelte +++ b/frontend/src/lib/components/common/table/Row.svelte @@ -10,6 +10,7 @@ interface Props { marked: string | undefined selected?: boolean + keyboardSelected?: boolean disabled?: boolean canFavorite?: boolean isSelectable?: boolean @@ -47,6 +48,7 @@ let { marked, selected = false, + keyboardSelected = false, disabled = false, canFavorite = true, isSelectable = false, @@ -73,6 +75,13 @@ : untrack(() => path) ?.split('/') ?.slice(-1)?.[0]) ?? '' + + let rowEl: HTMLDivElement | undefined = $state() + $effect(() => { + if (keyboardSelected) { + rowEl?.scrollIntoView({ block: 'nearest' }) + } + }) {#if href} @@ -88,11 +97,12 @@ > {/if}
0 ? '!rounded-none' : '', disabled ? 'opacity-25' : 'hover:bg-surface-hover', - selected ? 'bg-surface-accent-selected' : '' + selected ? 'bg-surface-accent-selected' : keyboardSelected ? 'bg-gray-200 dark:bg-gray-700' : '' )} style={depth > 0 ? `padding-left: ${depth * 32}px;` : ''} > @@ -105,6 +115,7 @@ {#if href} {@render rowContent()} @@ -131,7 +142,7 @@
{/if} -
+
{@render actions?.()}
diff --git a/frontend/src/lib/components/common/table/ScriptRow.svelte b/frontend/src/lib/components/common/table/ScriptRow.svelte index 91f8f2d5d0..3d0aec796b 100644 --- a/frontend/src/lib/components/common/table/ScriptRow.svelte +++ b/frontend/src/lib/components/common/table/ScriptRow.svelte @@ -62,6 +62,7 @@ depth?: number menuOpen?: boolean showEditButton?: boolean + keyboardSelected?: boolean } let { @@ -75,7 +76,8 @@ showCode, depth = 0, menuOpen = $bindable(false), - showEditButton = $bindable(true) + showEditButton = $bindable(true), + keyboardSelected = false }: Props = $props() const dispatch = createEventDispatcher() @@ -130,6 +132,7 @@ workspaceId={$workspaceStore ?? ''} canFavorite={!script.draft_only} {depth} + {keyboardSelected} > {#snippet badges()} {#if script.lock_error_logs} diff --git a/frontend/src/lib/components/home/Item.svelte b/frontend/src/lib/components/home/Item.svelte index 2e06e66417..6621e60fef 100644 --- a/frontend/src/lib/components/home/Item.svelte +++ b/frontend/src/lib/components/home/Item.svelte @@ -24,9 +24,16 @@ depth?: number showCode: (path: string, summary: string) => void showEditButton?: boolean + keyboardSelected?: boolean } - let { item, depth = 0, showCode, showEditButton = true }: Props = $props() + let { + item, + depth = 0, + showCode, + showEditButton = true, + keyboardSelected = false + }: Props = $props() {#if item.type == 'script'} @@ -46,6 +53,7 @@ bind:menuOpen {showCode} {showEditButton} + {keyboardSelected} /> {:else if item.type == 'flow'} {:else if item.type == 'app'} {:else if item.type == 'raw_app'} {/if} diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index f8bb688636..0248ceda1d 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -40,7 +40,7 @@ import Item from './Item.svelte' import TreeViewRoot from './TreeViewRoot.svelte' import Popover from '$lib/components/meltComponents/Popover.svelte' - import { getContext, untrack } from 'svelte' + import { getContext, tick, untrack } from 'svelte' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' import TextInput from '../text_input/TextInput.svelte' interface Props { @@ -326,9 +326,252 @@ ) ) let items = $derived(filter !== '' ? filteredItems : preFilteredItems) + let displayedItems = $derived((items ?? []).slice(0, nbDisplayed)) $effect(() => { items && resetScroll() }) + + let selectedIndex: number = $state(-1) + let hasMore = $derived(items != undefined && items.length > nbDisplayed) + let loadMoreIndex = $derived(displayedItems.length) + let loadMoreEl: HTMLButtonElement | undefined = $state() + let pendingAutoSelect = $state(true) + let firstWorkspaceRun = true + $effect(() => { + $workspaceStore + pendingAutoSelect = true + if (firstWorkspaceRun) { + firstWorkspaceRun = false + return + } + // On workspace switch, melt-ui restores focus to the workspace-picker trigger + // button asynchronously after the menu closes. Without overriding it, pressing + // an arrow key would re-open / re-highlight the workspace picker instead of + // moving the items-list selection. Run several times to win the focus race. + const focusSearch = () => { + const el = document.getElementById('home-search-input') as HTMLInputElement | null + el?.focus() + } + focusSearch() + const raf1 = requestAnimationFrame(() => { + focusSearch() + requestAnimationFrame(focusSearch) + }) + const timeoutId = setTimeout(focusSearch, 100) + return () => { + cancelAnimationFrame(raf1) + clearTimeout(timeoutId) + } + }) + $effect(() => { + filter + itemKind + ownerFilter + labelFilter + // Skip while pendingAutoSelect is true (initial load / workspace switch); + // the auto-select effect below will set the index once items appear. + if (!pendingAutoSelect) { + selectedIndex = -1 + } + }) + $effect(() => { + if (pendingAutoSelect && displayedItems.length > 0) { + selectedIndex = 0 + pendingAutoSelect = false + } + }) + $effect(() => { + const max = hasMore ? displayedItems.length : displayedItems.length - 1 + if (selectedIndex > max) { + selectedIndex = max + } + }) + $effect(() => { + if (hasMore && selectedIndex === loadMoreIndex) { + loadMoreEl?.scrollIntoView({ block: 'nearest' }) + } + }) + // Capture-phase listener so we run before melt-ui's button keydown handlers + // (e.g. ArrowDown on the dropdown trigger would otherwise open the menu). + $effect(() => { + window.addEventListener('keydown', handleGlobalKeydown, true) + return () => window.removeEventListener('keydown', handleGlobalKeydown, true) + }) + + function loadMoreAndPreselectFirstNew() { + const previousNbDisplayed = nbDisplayed + nbDisplayed += 30 + selectedIndex = previousNbDisplayed + } + + function getSelectedRowActionButtons(): HTMLElement[] { + const anchor = document.querySelector('a[data-row-keyboard-selected="true"]') + const actions = anchor?.parentElement?.querySelector('[data-row-actions]') + return actions ? Array.from(actions.querySelectorAll('button, a[href]')) : [] + } + + function handleGlobalKeydown(e: KeyboardEvent) { + if (treeView) return + const target = e.target as HTMLElement | null + + // When focus is inside a row's action buttons, handle arrow keys ourselves: + // - Left/Right cycle between buttons (Left from the first returns to search). + // - Up/Down move to the same-position button on the previous/next row. + // All other keys pass through so Enter/Space activate the focused button normally. + // This must run BEFORE the skipSelector check, since the dropdown ellipsis + // trigger carries [data-menu] (which would otherwise filter the event out). + // Up/Down also need stopImmediatePropagation so melt-ui's dropdown trigger + // doesn't open the menu (its default ArrowDown behavior). + const actionsContainer = target?.closest('[data-row-actions]') + if (actionsContainer) { + if ( + e.key !== 'ArrowRight' && + e.key !== 'ArrowLeft' && + e.key !== 'ArrowUp' && + e.key !== 'ArrowDown' + ) + return + const buttons = Array.from(actionsContainer.querySelectorAll('button, a[href]')) + const currentIdx = buttons.indexOf(target as HTMLElement) + if (currentIdx < 0) return + if (e.key === 'ArrowRight') { + if (currentIdx < buttons.length - 1) { + e.preventDefault() + buttons[currentIdx + 1].focus() + } + } else if (e.key === 'ArrowLeft') { + e.preventDefault() + if (currentIdx > 0) { + buttons[currentIdx - 1].focus() + } else { + ;(document.getElementById('home-search-input') as HTMLInputElement | null)?.focus() + } + } else { + // ArrowUp / ArrowDown: move to same-position button on prev/next row. + e.preventDefault() + e.stopImmediatePropagation() + if (selectedIndex < 0 || selectedIndex >= displayedItems.length) return + const newIndex = + e.key === 'ArrowDown' + ? Math.min(selectedIndex + 1, displayedItems.length - 1) + : Math.max(selectedIndex - 1, 0) + if (newIndex === selectedIndex) return + selectedIndex = newIndex + tick().then(() => { + const newButtons = getSelectedRowActionButtons() + if (newButtons.length === 0) return + const targetIdx = Math.min(currentIdx, newButtons.length - 1) + newButtons[targetIdx]?.focus() + }) + } + return + } + + // Inside an open dropdown menu: ArrowUp on first item / ArrowDown on last item + // closes the menu (so users can leave with arrows instead of needing Escape). + // Other arrow keys fall through to melt-ui's default cycle. + const menuItem = target?.closest('[role="menuitem"]') + if (menuItem) { + if (e.key === 'ArrowUp' || e.key === 'ArrowDown') { + const menu = menuItem.closest('[role="menu"]') + if (menu) { + const items = Array.from(menu.querySelectorAll('[role="menuitem"]')) + const idx = items.indexOf(menuItem) + const isFirst = idx === 0 + const isLast = idx === items.length - 1 + if ((e.key === 'ArrowUp' && isFirst) || (e.key === 'ArrowDown' && isLast)) { + e.preventDefault() + e.stopImmediatePropagation() + menuItem.dispatchEvent( + new KeyboardEvent('keydown', { key: 'Escape', bubbles: true, cancelable: true }) + ) + } + } + } + return + } + + const skipSelector = + '[role="menu"], [role="menuitem"], [role="dialog"], [role="listbox"], [role="combobox"], [aria-expanded="true"], [data-menu]' + if (target) { + const tag = target.tagName + const isEditable = + tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || target.isContentEditable + const isOurSearch = target.id === 'home-search-input' + if (isEditable && !isOurSearch) return + if (target.closest(skipSelector)) return + } + const active = document.activeElement as HTMLElement | null + if (active?.closest(skipSelector)) return + + // ArrowRight from search input / body → focus first action button of selected row. + // Guard: if cursor is in the middle of typed search text, let the cursor move. + if (e.key === 'ArrowRight') { + if (target?.id === 'home-search-input') { + const inp = target as HTMLInputElement + if (inp.value.length > 0 && inp.selectionEnd !== inp.value.length) return + } + if (selectedIndex < 0 || selectedIndex >= displayedItems.length) return + const buttons = getSelectedRowActionButtons() + if (buttons.length > 0) { + e.preventDefault() + buttons[0].focus() + } + return + } + // ArrowLeft from search input with cursor at start: no-op (let default handle). + if (e.key === 'ArrowLeft') { + if (target?.id === 'home-search-input') { + const inp = target as HTMLInputElement + if (inp.value.length > 0 && inp.selectionStart !== 0) return + } + return + } + + if (e.key === 'ArrowDown') { + if (displayedItems.length === 0) return + e.preventDefault() + if (selectedIndex === -1) { + selectedIndex = 0 + } else if (selectedIndex === loadMoreIndex && hasMore) { + selectedIndex = 0 + } else if (selectedIndex === displayedItems.length - 1) { + selectedIndex = hasMore ? loadMoreIndex : 0 + } else { + selectedIndex = selectedIndex + 1 + } + } else if (e.key === 'ArrowUp') { + if (displayedItems.length === 0) return + e.preventDefault() + if (selectedIndex === -1) { + selectedIndex = displayedItems.length - 1 + } else if (selectedIndex === loadMoreIndex && hasMore) { + selectedIndex = displayedItems.length - 1 + } else if (selectedIndex === 0) { + selectedIndex = hasMore ? loadMoreIndex : displayedItems.length - 1 + } else { + selectedIndex = selectedIndex - 1 + } + } else if (e.key === 'Enter') { + if (selectedIndex === loadMoreIndex && hasMore) { + e.preventDefault() + loadMoreAndPreselectFirstNew() + } else if (selectedIndex >= 0 && selectedIndex < displayedItems.length) { + const anchor = document.querySelector( + 'a[data-row-keyboard-selected="true"]' + ) + if (anchor) { + e.preventDefault() + anchor.click() + } + } + } else if (e.key === 'Escape') { + if (selectedIndex !== -1) { + e.preventDefault() + selectedIndex = -1 + } + } + } $effect(() => { storeLocalSetting(TREE_VIEW_SETTING_NAME, treeView ? 'true' : undefined) }) @@ -572,7 +815,7 @@ /> {:else}
- {#each (items ?? []).slice(0, nbDisplayed) as item (item.type + '/' + item.path + (item.hash ? '/' + item.hash : ''))} + {#each displayedItems as item, i (item.type + '/' + item.path + (item.hash ? '/' + item.hash : ''))} loadScripts(includeWithoutMain)} @@ -587,6 +830,7 @@ }} {showCode} showEditButton={showEditButtons} + keyboardSelected={selectedIndex === i} /> {/each}
@@ -594,7 +838,11 @@ {nbDisplayed} items out of {items.length} From f07f19ebe7530934eb20e89a35ac31756d7d4fa9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 5 May 2026 21:20:40 +0000 Subject: [PATCH 34/56] chore(main): release 1.696.0 (#9040) * chore(main): release 1.696.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 19 ++ backend/Cargo.lock | 208 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++-- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 54 ++++- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 206 insertions(+), 151 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f12091b720..7048785010 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## [1.696.0](https://github.com/windmill-labs/windmill/compare/v1.695.0...v1.696.0) (2026-05-05) + + +### Features + +* add ai chat resource action buttons ([#9016](https://github.com/windmill-labs/windmill/issues/9016)) ([502a029](https://github.com/windmill-labs/windmill/commit/502a02998685308e82fab95bae7d3c14efd77d6a)) +* add wac ai context for frontend chat ([#9021](https://github.com/windmill-labs/windmill/issues/9021)) ([0d0557f](https://github.com/windmill-labs/windmill/commit/0d0557fc9dc5addee887911ddfc0fd08a09bc92e)) +* **cli:** add --as-superadmin flag to workspace list-remote ([#9043](https://github.com/windmill-labs/windmill/issues/9043)) ([66c9063](https://github.com/windmill-labs/windmill/commit/66c90639191a77eb4f19da092167384565edb9b3)) + + +### Bug Fixes + +* **cli:** resolve cross-folder relative imports during lockgen on fresh DB ([#9048](https://github.com/windmill-labs/windmill/issues/9048)) ([40dbab5](https://github.com/windmill-labs/windmill/commit/40dbab531e5166b894f3f94b0d72b2ac456c0097)) +* **flows:** inherit flow_env in sub-flow predicates ([#9042](https://github.com/windmill-labs/windmill/issues/9042)) ([6e5a21a](https://github.com/windmill-labs/windmill/commit/6e5a21a9c7b5db77d325b5916a9ab8799a2eb6e7)) +* navigate home arrows ([#9024](https://github.com/windmill-labs/windmill/issues/9024)) ([c1e52ea](https://github.com/windmill-labs/windmill/commit/c1e52eab09794746bea7dc9adb94552641f87d5b)) +* open job detail header path links in a new tab ([#9039](https://github.com/windmill-labs/windmill/issues/9039)) ([fe68c06](https://github.com/windmill-labs/windmill/commit/fe68c066004d860088e09be32e7ff2e7438f78c4)) +* **rust-client:** re-export models module from wmill crate ([#9038](https://github.com/windmill-labs/windmill/issues/9038)) ([ca6efbf](https://github.com/windmill-labs/windmill/commit/ca6efbff74e7d7e85174b1d8394af79eda7d6535)) +* **windmill-utils-internal:** move config to subpath export ([#9045](https://github.com/windmill-labs/windmill/issues/9045)) ([b86f896](https://github.com/windmill-labs/windmill/commit/b86f8960fcd8a66bc6849638178ef45ac49e06f1)) + ## [1.695.0](https://github.com/windmill-labs/windmill/compare/v1.694.0...v1.695.0) (2026-05-04) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index ddcb9848f9..c188373029 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1221,7 +1221,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "h2 0.3.27", - "h2 0.4.13", + "h2 0.4.14", "http 0.2.12", "http 1.4.0", "http-body 0.4.6", @@ -4001,7 +4001,7 @@ dependencies = [ "deno_tls", "dyn-clone", "error_reporter", - "h2 0.4.13", + "h2 0.4.14", "hickory-resolver", "http 1.4.0", "http-body-util", @@ -4282,7 +4282,7 @@ dependencies = [ "elliptic-curve", "errno", "faster-hex", - "h2 0.4.13", + "h2 0.4.14", "hkdf", "http 1.4.0", "http-body-util", @@ -4738,7 +4738,7 @@ dependencies = [ "deno_permissions", "deno_tls", "fastwebsockets", - "h2 0.4.13", + "h2 0.4.14", "http 1.4.0", "http-body-util", "hyper 1.9.0", @@ -6558,9 +6558,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -7041,7 +7041,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.13", + "h2 0.4.14", "http 1.4.0", "http-body 1.0.1", "httparse", @@ -7539,16 +7539,6 @@ dependencies = [ "serde", ] -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is-macro" version = "0.3.7" @@ -8168,7 +8158,7 @@ dependencies = [ "bitflags 2.9.4", "libc", "plain", - "redox_syscall 0.7.4", + "redox_syscall 0.7.5", ] [[package]] @@ -10259,18 +10249,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" dependencies = [ "proc-macro2", "quote", @@ -11162,9 +11152,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" +checksum = "4666a1a60d8412eab19d94f6d13dcc9cea0a5ef4fdf6a5db306537413c661b1b" dependencies = [ "bitflags 2.9.4", ] @@ -11282,7 +11272,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.13", + "h2 0.4.14", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -11330,7 +11320,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.13", + "h2 0.4.14", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -14721,7 +14711,7 @@ dependencies = [ "base64 0.22.1", "bytes", "flate2", - "h2 0.4.13", + "h2 0.4.14", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -14753,7 +14743,7 @@ dependencies = [ "axum 0.8.4", "base64 0.22.1", "bytes", - "h2 0.4.13", + "h2 0.4.14", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -14831,9 +14821,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "a28f0d049ccfaa566e14e9663d304d8577427b368cb4710a20528690287a738b" dependencies = [ "async-compression", "base64 0.22.1", @@ -14844,7 +14834,6 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", - "iri-string", "mime", "pin-project-lite", "tokio", @@ -14853,6 +14842,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -16019,7 +16009,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-nats", @@ -16100,7 +16090,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.695.0" +version = "1.696.0" dependencies = [ "async-trait", "aws-config", @@ -16124,7 +16114,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16137,7 +16127,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "argon2", @@ -16280,7 +16270,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16303,7 +16293,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16316,7 +16306,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16342,7 +16332,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.695.0" +version = "1.696.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -16352,7 +16342,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16369,7 +16359,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "base64 0.22.1", @@ -16391,7 +16381,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16414,7 +16404,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16430,7 +16420,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16451,7 +16441,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16472,7 +16462,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16486,7 +16476,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-nats", @@ -16518,7 +16508,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16543,7 +16533,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "flate2", @@ -16561,7 +16551,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16583,7 +16573,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16603,7 +16593,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16633,7 +16623,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16661,7 +16651,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.695.0" +version = "1.696.0" dependencies = [ "lazy_static", "serde", @@ -16673,7 +16663,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.695.0" +version = "1.696.0" dependencies = [ "argon2", "axum 0.8.4", @@ -16698,7 +16688,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16712,7 +16702,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.695.0" +version = "1.696.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16745,7 +16735,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.695.0" +version = "1.696.0" dependencies = [ "chrono", "lazy_static", @@ -16759,7 +16749,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16778,7 +16768,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.695.0" +version = "1.696.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -16879,7 +16869,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.695.0" +version = "1.696.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -16898,7 +16888,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.695.0" +version = "1.696.0" dependencies = [ "regex", "serde", @@ -16913,7 +16903,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16937,7 +16927,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "futures", @@ -16954,7 +16944,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.695.0" +version = "1.696.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16970,7 +16960,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -16991,7 +16981,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17022,7 +17012,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "arc-swap", @@ -17047,7 +17037,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-stream", @@ -17081,7 +17071,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "futures", @@ -17099,7 +17089,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.695.0" +version = "1.696.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -17108,7 +17098,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "lazy_static", @@ -17120,7 +17110,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde_json", @@ -17132,7 +17122,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "gosyn", @@ -17144,7 +17134,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "lazy_static", @@ -17156,7 +17146,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde_json", @@ -17168,7 +17158,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "nu-parser", @@ -17179,7 +17169,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -17190,7 +17180,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -17202,7 +17192,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "rustpython-ast", @@ -17213,7 +17203,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-recursion", @@ -17235,7 +17225,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde_json", @@ -17247,7 +17237,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "lazy_static", @@ -17261,7 +17251,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -17278,7 +17268,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "lazy_static", @@ -17291,7 +17281,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde", @@ -17303,7 +17293,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "lazy_static", @@ -17321,7 +17311,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -17337,7 +17327,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "rustpython-ast", @@ -17353,7 +17343,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde", @@ -17364,7 +17354,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-recursion", @@ -17401,7 +17391,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "const_format", @@ -17439,7 +17429,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.695.0" +version = "1.696.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17450,7 +17440,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-recursion", @@ -17480,7 +17470,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17504,7 +17494,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17537,7 +17527,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17570,7 +17560,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17590,7 +17580,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17624,7 +17614,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17660,7 +17650,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17683,7 +17673,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17707,7 +17697,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-nats", @@ -17731,7 +17721,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17766,7 +17756,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17794,7 +17784,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-trait", @@ -17817,7 +17807,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17836,7 +17826,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-once-cell", @@ -17948,7 +17938,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.695.0" +version = "1.696.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index fd6f00e68b..6792679886 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.695.0" +version = "1.696.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.695.0" +version = "1.696.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 6687a77c03..45d0fe8fff 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6183,7 +6183,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.695.0" +version = "1.696.0" dependencies = [ "aho-corasick", "anyhow", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.695.0" +version = "1.696.0" dependencies = [ "proc-macro2", "quote", @@ -6275,7 +6275,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.695.0" +version = "1.696.0" dependencies = [ "convert_case", "serde", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "lazy_static", @@ -6296,7 +6296,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde_json", @@ -6308,7 +6308,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "gosyn", @@ -6320,7 +6320,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "lazy_static", @@ -6332,7 +6332,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde_json", @@ -6344,7 +6344,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "nu-parser", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "async-recursion", @@ -6411,7 +6411,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde_json", @@ -6423,7 +6423,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "lazy_static", @@ -6437,7 +6437,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "convert_case", @@ -6454,7 +6454,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "lazy_static", @@ -6467,7 +6467,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "lazy_static", @@ -6497,7 +6497,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6513,7 +6513,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6529,7 +6529,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6561,7 +6561,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "serde", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.695.0" +version = "1.696.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index a77fcc6436..47adc66cb5 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.695.0" +version = "1.696.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index f44b5fac5e..644e01cbb9 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.695.0 + version: 1.696.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 6707029f16..73e51d884d 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.695.0"; +export const VERSION = "v1.696.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index fb030e3347..aaa87937b6 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -78,7 +78,7 @@ export { token, }; -export const VERSION = "1.695.0"; +export const VERSION = "1.696.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 76002aba4a..53a178dfca 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.695.0", + "version": "1.696.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.695.0", + "version": "1.696.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -844,6 +844,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.0.tgz", "integrity": "sha512-0DQ98G9ZQZOxfUcQn1waV2yS8aWdZ6kJMbYCJB3oUBecjWYO1fqJ+a1DRfPF3O5JEkwqwP1A9QEN/9mYm2Yd0w==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -855,6 +856,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.0.tgz", "integrity": "sha512-QN75eB0IH2ywSpRpNddCRfQIhmJYBCJ1x5Lb3IscKAL8bMnVAKnRg8dCoXbHzVLLH7P38N2Z3mtulB7W0J0FKw==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -865,6 +867,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.0.tgz", "integrity": "sha512-N10dEJNSsUx41Z6pZsXU8FjPjpBEplgH24sfkmITrBED1/U2Esum9F3lfLrMjKHHjmi557zQn7kR9R+XWXu5Rg==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1354,6 +1357,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz", "integrity": "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1510,6 +1514,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1526,6 +1531,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1542,6 +1548,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1558,6 +1565,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1574,6 +1582,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1590,6 +1599,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1606,6 +1616,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1622,6 +1633,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1638,6 +1650,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1654,6 +1667,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1670,6 +1684,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1686,6 +1701,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1702,6 +1718,7 @@ "cpu": [ "wasm32" ], + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1718,6 +1735,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1734,6 +1752,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2039,6 +2058,7 @@ "version": "0.10.1", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -6814,7 +6834,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "devOptional": true, + "dev": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -7313,6 +7333,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7333,6 +7354,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7353,6 +7375,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7373,6 +7396,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7393,6 +7417,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7413,6 +7438,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7433,6 +7459,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7453,6 +7480,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7473,6 +7501,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7493,6 +7522,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7513,6 +7543,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -12081,6 +12112,21 @@ } } }, + "node_modules/svelte-check/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -12811,7 +12857,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/frontend/package.json b/frontend/package.json index 98ff10ae9d..5bf6e54088 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.695.0", + "version": "1.696.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 9b60f26f0a..e6064a7d20 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.695.0" +wmill = ">=1.696.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 910df51d99..8f8819e511 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.695.0 + version: 1.696.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 9811b57cf3..918e4afa5f 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.695.0' + ModuleVersion = '1.696.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 3ff1345b0e..8c2d924858 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.695.0" +version = "1.696.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index bcbb5fbc5a..21a0d9396d 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.695.0", + "version": "1.696.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 75b89c0ba2..8e170e0daa 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.695.0", + "version": "1.696.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index af443c10ea..22f9e1c0a1 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.695.0 +1.696.0 From f4553e8e7919b115a4239a62ee4587347cc82bb8 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 5 May 2026 21:49:56 +0000 Subject: [PATCH 35/56] fix(workspaces): validate fork id as a git branch name component (#9049) --- .../windmill-api-workspaces/src/workspaces.rs | 16 +-- backend/windmill-common/src/workspaces.rs | 109 ++++++++++++++++++ 2 files changed, 115 insertions(+), 10 deletions(-) diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 9ca9a0dd84..a4173fc612 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -40,9 +40,9 @@ use windmill_common::workspaces::GitRepositorySettings; #[cfg(feature = "enterprise")] use windmill_common::workspaces::WorkspaceDeploymentUISettings; use windmill_common::workspaces::{ - check_user_against_rule, get_datatable_resource_from_db_unchecked, DataTable, - DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, ProtectionRules, - ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, WM_FORK_PREFIX, + check_user_against_rule, get_datatable_resource_from_db_unchecked, validate_fork_workspace_id, + DataTable, DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, + ProtectionRules, ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, }; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; @@ -4776,6 +4776,8 @@ async fn create_workspace_fork_branch( return Err(Error::PermissionDenied(msg)); } + validate_fork_workspace_id(&nw.id)?; + Ok(Json( handle_fork_branch_creation(&authed.email, &authed.username, &db, &w_id, &nw.id).await?, )) @@ -4935,13 +4937,7 @@ async fn create_workspace_fork( let mut tx: Transaction<'_, Postgres> = db.begin().await?; - // Generate unique forked workspace ID with wm-fork prefix - if !nw.id.starts_with(WM_FORK_PREFIX) { - return Err(Error::BadRequest(format!( - "The id `{}` is invalid for a forked workspace. It should be prefixed by {}", - nw.id, WM_FORK_PREFIX - ))); - } + validate_fork_workspace_id(&nw.id)?; let forked_id = nw.id; diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 77e69a3340..74ea9f9c9e 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -163,6 +163,65 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28217/sync-script-to-git-repo /// fork of another workspace. pub const WM_FORK_PREFIX: &str = "wm-fork-"; +/// Validate that a fork workspace id is safe to interpolate into a git branch name. +/// +/// The id is appended verbatim to a branch like `wm-fork//`, +/// so it must satisfy `git check-ref-format` rules. We validate synchronously at the API +/// layer because the actual branch creation runs in a deferred git-sync worker job — without +/// this check, the API returns 200 and the failure only surfaces later in the worker. +pub fn validate_fork_workspace_id(id: &str) -> error::Result<()> { + if !id.starts_with(WM_FORK_PREFIX) { + return Err(Error::BadRequest(format!( + "The id `{}` is invalid for a forked workspace. It should be prefixed by {}", + id, WM_FORK_PREFIX + ))); + } + + let reject = |reason: &str| { + Err::<(), _>(Error::BadRequest(format!( + "Fork workspace id `{}` is invalid: {} (must be a valid git branch name component)", + id, reason + ))) + }; + + if id.ends_with('.') { + return reject("cannot end with '.'"); + } + if id.ends_with(".lock") { + return reject("cannot end with '.lock'"); + } + if id.contains("..") { + return reject("cannot contain '..'"); + } + if id.contains("@{") { + return reject("cannot contain '@{'"); + } + if id.contains("//") { + return reject("cannot contain '//'"); + } + for ch in id.chars() { + match ch { + ':' | '~' | '^' | '?' | '*' | '[' | '\\' | ' ' => { + return reject(&format!("contains forbidden character '{}'", ch)); + } + c if c.is_ascii_control() || c == '\u{7f}' => { + return reject("contains a control character"); + } + _ => {} + } + } + // Each slash-separated component cannot start with '.' or end with '.lock'. + for component in id.split('/') { + if component.starts_with('.') { + return reject("a path component cannot start with '.'"); + } + if component.ends_with(".lock") { + return reject("a path component cannot end with '.lock'"); + } + } + Ok(()) +} + #[derive(Serialize, Deserialize, Debug)] pub struct GitRepositorySettings { #[serde(skip_serializing_if = "Option::is_none")] @@ -666,3 +725,53 @@ async fn transform_json_unchecked( Ok(value) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_validate_fork_workspace_id_accepts_valid() { + validate_fork_workspace_id("wm-fork-test-allow").unwrap(); + validate_fork_workspace_id("wm-fork-my_workspace.42").unwrap(); + validate_fork_workspace_id("wm-fork-a").unwrap(); + } + + #[test] + fn test_validate_fork_workspace_id_rejects_missing_prefix() { + assert!(validate_fork_workspace_id("not-a-fork").is_err()); + assert!(validate_fork_workspace_id("wm-fork").is_err()); + } + + #[test] + fn test_validate_fork_workspace_id_rejects_git_unsafe_chars() { + for bad in [ + "wm-fork-test:allow", + "wm-fork-test allow", + "wm-fork-test~allow", + "wm-fork-test^allow", + "wm-fork-test?allow", + "wm-fork-test*allow", + "wm-fork-test[allow", + "wm-fork-test\\allow", + "wm-fork-test\nallow", + ] { + assert!( + validate_fork_workspace_id(bad).is_err(), + "expected `{}` to be rejected", + bad + ); + } + } + + #[test] + fn test_validate_fork_workspace_id_rejects_git_unsafe_sequences() { + assert!(validate_fork_workspace_id("wm-fork-foo..bar").is_err()); + assert!(validate_fork_workspace_id("wm-fork-foo@{bar").is_err()); + assert!(validate_fork_workspace_id("wm-fork-foo//bar").is_err()); + assert!(validate_fork_workspace_id("wm-fork-foo.").is_err()); + assert!(validate_fork_workspace_id("wm-fork-foo.lock").is_err()); + assert!(validate_fork_workspace_id("wm-fork-foo/.bar").is_err()); + assert!(validate_fork_workspace_id("wm-fork-foo/bar.lock").is_err()); + } +} From eebaab9c87f975b70049e118a08665fc21653b13 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 6 May 2026 10:00:46 +0000 Subject: [PATCH 36/56] fix(bun): propagate non-zero exit from generate_bun_bundle on no-DB path (#9051) --- backend/src/main.rs | 2 - backend/tests/bun_jobs.rs | 215 ++++++++++++++++++ backend/windmill-worker/loader_builder.bun.js | 8 +- backend/windmill-worker/src/bun_executor.rs | 65 ++++-- backend/windmill-worker/src/lib.rs | 7 +- 5 files changed, 278 insertions(+), 19 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index ce09c7f25f..3f3a9b0e29 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -305,8 +305,6 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { ) .await?; - let _ = windmill_common::worker::write_file(&job_dir, "main.js", &res.content)?; - if let Err(e) = windmill_worker::prebundle_bun_script( &res.content, &lock, diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index a791060a2c..fa15866bca 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -956,6 +956,221 @@ export function main() { Ok(()) } +// ============================================================================ +// Bundle Wrapper Safety Tests +// ============================================================================ + +/// Regression test for the "TS source ends up in the bun bundle cache" bug. +/// +/// The wrapper-side hardening: `node_builder.ts` discarded `Bun.build`'s +/// return value, so any silent-failure mode (`success: false` without +/// throwing — `throw: false`, or a future Bun where defaults change) made +/// the wrapper exit 0 even though no `main.js` was written. Pair that with +/// a pre-existing `main.js` containing raw TypeScript and `save_cache` +/// happily copied that TS into the bundle cache; the worker later choked +/// on `type GpgKey = {`. +/// +/// This test patches `node_builder.ts` to force the silent-failure shape +/// and asserts that our wrapper now refuses to silently succeed — bun must +/// exit non-zero so prebundling fails loudly instead of writing TypeScript +/// into the bundle cache. +#[test] +fn test_bun_bundle_wrapper_catches_silent_failure() { + use std::process::Command; + use windmill_worker::{build_loader, LoaderMode, BUN_PATH}; + + let temp_dir = tempfile::tempdir().unwrap(); + let dir = temp_dir.path(); + let dir_str = dir.to_str().unwrap(); + + // Script imports a package that won't exist in node_modules. + std::fs::write( + dir.join("main.ts"), + r#" +import x from "definitely-not-a-real-pkg-windmill-test"; +export function main() { return x; } +"#, + ) + .unwrap(); + + // Generate the real node_builder.ts via the production code path. + tokio::runtime::Runtime::new() + .unwrap() + .block_on(build_loader( + dir_str, + "http://localhost:8000", + "test_token", + "test-workspace", + "f/test/script", + LoaderMode::BunBundle, + &None, + )) + .expect("build_loader failed"); + + // Force the silent-failure shape by injecting `throw: false`. The + // wrapper's pre-fix `try/catch` would have swallowed this; the fixed + // wrapper inspects `result.success` and `result.outputs` and exits 1. + let path = dir.join("node_builder.ts"); + let original = std::fs::read_to_string(&path).unwrap(); + let patched = original.replace( + "external: [\"electron\"],", + "external: [\"electron\"], throw: false,", + ); + assert_ne!( + original, patched, + "expected to find Bun.build options block to patch; node_builder.ts template changed?" + ); + std::fs::write(&path, patched).unwrap(); + + // Pre-seed main.js with raw TypeScript (mimics the historical + // pre-write that originally seeded the bug). + std::fs::write( + dir.join("main.js"), + "type GpgKey = { email: string };\nexport const main = (): GpgKey => ({ email: \"\" });\n", + ) + .unwrap(); + + let output = Command::new(BUN_PATH.as_str()) + .args(["run", path.to_str().unwrap()]) + .current_dir(dir) + .output() + .expect("Failed to run bun"); + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "node_builder.ts must exit non-zero when Bun.build silently fails to write a bundle.\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + assert!( + stdout.contains("Failed to build node bundle"), + "expected diagnostic in stdout, got:\n{stdout}" + ); +} + +/// Regression test for the actual root cause of the "TS source in bundle +/// cache" bug: `generate_bun_bundle` was awaiting `child_process.wait()` +/// without checking the exit code on the no-DB path (used by Docker-build +/// `windmill cache hubPaths.json`). bun would exit 1 after Bun.build threw, +/// `wait().await?` propagated only IO errors, and `generate_bun_bundle` +/// returned `Ok(())`. `save_cache` then copied a stale `main.js` (raw TS +/// source) straight into the bundle cache. +/// +/// This test runs `generate_bun_bundle` with `db: None` against a `node_builder.ts` +/// that calls `process.exit(1)`, and asserts the function now returns an error. +#[test] +fn test_generate_bun_bundle_propagates_exit_status() { + use windmill_worker::{generate_bun_bundle, get_common_bun_proc_envs}; + + let temp_dir = tempfile::tempdir().unwrap(); + let dir = temp_dir.path(); + let dir_str = dir.to_str().unwrap(); + + // node_builder.ts that exits 1, mimicking what bun does when Bun.build throws. + std::fs::write( + dir.join("node_builder.ts"), + "console.log('simulated bun build failure');\nprocess.exit(1);\n", + ) + .unwrap(); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let envs = runtime.block_on(get_common_bun_proc_envs(None)); + + let result = runtime.block_on(generate_bun_bundle( + dir_str, + "test-workspace", + &uuid::Uuid::new_v4(), + "test-worker", + None, // db: None — this is the cache_hub_scripts path that had the bug + None, + &mut 0, + &mut None, + &envs, + &mut None, + )); + + assert!( + result.is_err(), + "generate_bun_bundle must surface bun's non-zero exit on the no-DB path. \ + If it returns Ok(()) when bun exited 1, save_cache will silently cache stale main.js content." + ); + let err_msg = format!("{:?}", result.unwrap_err()); + assert!( + err_msg.contains("non-zero status"), + "expected exit-status error, got: {err_msg}" + ); +} + +/// Regression test for the install_bun_lockfile no-DB path: same code shape as +/// `generate_bun_bundle` (site 3 of the original bug) — `wait().await?` ignored +/// non-zero bun exits. A `bun install` failure (e.g. malformed package.json) +/// must now surface as an error so callers don't proceed with a half-installed +/// node_modules. +#[test] +fn test_install_bun_lockfile_propagates_exit_status() { + use windmill_worker::{get_common_bun_proc_envs, install_bun_lockfile}; + let temp_dir = tempfile::tempdir().unwrap(); + let dir = temp_dir.path(); + let dir_str = dir.to_str().unwrap(); + // Malformed package.json -> bun install fails with exit 1. + std::fs::write(dir.join("package.json"), "this is not valid json").unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let envs = runtime.block_on(get_common_bun_proc_envs(None)); + let result = runtime.block_on(install_bun_lockfile( + &mut 0, + &mut None, + &uuid::Uuid::new_v4(), + "test-workspace", + None, // db: None — no-DB path that had the bug + dir_str, + "test-worker", + envs, + false, // npm_mode + &mut None, + true, // quiet + )); + assert!( + result.is_err(), + "install_bun_lockfile must surface bun's non-zero exit on the no-DB path" + ); + let err_msg = format!("{:?}", result.unwrap_err()); + assert!( + err_msg.contains("non-zero status"), + "expected exit-status error, got: {err_msg}" + ); +} + +/// Regression test for the post-bundle existence check in `prebundle_bun_script` +/// and `handle_bun_job`. Both call sites guard against the case where +/// `generate_bun_bundle` returns `Ok(())` but `main.js` was never written — +/// the upstream wait-status fix is the primary defense, this is the catch-all +/// for any other silent-failure mode (Bun output-naming change, custom plugin +/// swallowing the build, etc.). Without this check, `save_cache` would +/// happily copy whatever's at the bundle path (often raw TypeScript that some +/// other code path left there). +#[test] +fn test_ensure_bundle_output_exists_rejects_missing_file() { + use windmill_worker::ensure_bundle_output_exists; + let temp_dir = tempfile::tempdir().unwrap(); + let dir = temp_dir.path(); + let missing = dir.join("main.js").to_str().unwrap().to_string(); + + let result = ensure_bundle_output_exists(&missing); + assert!( + result.is_err(), + "ensure_bundle_output_exists must reject when the bundle file is missing" + ); + let err_msg = format!("{:?}", result.unwrap_err()); + assert!( + err_msg.contains("bun bundle output missing"), + "expected 'bun bundle output missing' in error, got: {err_msg}" + ); + + // Sanity: when the file does exist, it returns Ok. + std::fs::write(&missing, "// @bun\n").unwrap(); + assert!(ensure_bundle_output_exists(&missing).is_ok()); +} + // ============================================================================ // Dedicated Worker Protocol Tests // ============================================================================ diff --git a/backend/windmill-worker/loader_builder.bun.js b/backend/windmill-worker/loader_builder.bun.js index 5ca1b22ab2..d2206b398b 100644 --- a/backend/windmill-worker/loader_builder.bun.js +++ b/backend/windmill-worker/loader_builder.bun.js @@ -1,5 +1,6 @@ +let buildResult; try { - await Bun.build({ + buildResult = await Bun.build({ entrypoints: ["./main.ts"], outdir: "./out", plugins: [p], @@ -17,6 +18,11 @@ try { console.log(err); process.exit(1); } +if (!buildResult?.success || !(buildResult.outputs?.length > 0)) { + for (const log of buildResult?.logs ?? []) console.log(log); + console.log("Failed to build bundle: success=" + buildResult?.success + ", outputs=" + (buildResult?.outputs?.length ?? 0)); + process.exit(1); +} const fs = require("fs/promises"); diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index be842e28b4..82d59d44b1 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -471,7 +471,12 @@ pub async fn gen_bun_lockfile( } result?; } else { - Box::into_pin(child_process.wait()).await?; + let status = Box::into_pin(child_process.wait()).await?; + if !status.success() { + return Err(error::Error::ExecutionErr(format!( + "bun build exited with non-zero status: {status:?}" + ))); + } } let new_package_json = read_file_content(&format!("{job_dir}/package.json")).await?; @@ -777,7 +782,12 @@ pub async fn install_bun_lockfile( } result?; } else { - Box::into_pin(child_process.wait()).await?; + let status = Box::into_pin(child_process.wait()).await?; + if !status.success() { + return Err(error::Error::ExecutionErr(format!( + "bun install exited with non-zero status: {status:?}" + ))); + } } if has_file { @@ -838,8 +848,9 @@ try {{ }} catch (e) {{ }} +let result; try {{ - await Bun.build({{ + result = await Bun.build({{ entrypoints: ["{job_dir_js}/wrapper.mjs"], outdir: "./", target: "node", @@ -852,6 +863,11 @@ try {{ console.log("Failed to build node bundle"); process.exit(1); }} +if (!result?.success || !(result.outputs?.length > 0)) {{ + for (const log of result?.logs ?? []) console.log(log); + console.log("Failed to build node bundle: success=" + result?.success + ", outputs=" + (result?.outputs?.length ?? 0)); + process.exit(1); +}} "# ), )?; @@ -880,8 +896,9 @@ plugin(p) r#" {loader} +let result; try {{ - await Bun.build({{ + result = await Bun.build({{ entrypoints: ["{job_dir_js}/main.ts"], outdir: "./", target: "{}", @@ -898,6 +915,11 @@ try {{ console.log("Failed to build node bundle"); process.exit(1); }} +if (!result?.success || !(result.outputs?.length > 0)) {{ + for (const log of result?.logs ?? []) console.log(log); + console.log("Failed to build node bundle: success=" + result?.success + ", outputs=" + (result?.outputs?.length ?? 0)); + process.exit(1); +}} "#, if mode == LoaderMode::BunBundle { "bun" @@ -1008,7 +1030,12 @@ pub async fn generate_bun_bundle( ) .await?; } else { - Box::into_pin(child_process.wait()).await?; + let status = Box::into_pin(child_process.wait()).await?; + if !status.success() { + return Err(error::Error::ExecutionErr(format!( + "bun build exited with non-zero status: {status:?}" + ))); + } } Ok(()) } @@ -1137,6 +1164,9 @@ pub async fn prebundle_bun_script( content = format!("export {{ WorkflowCtx, StepSuspend, setWorkflowCtx }} from \"windmill-client\";\n{content}"); } write_file(job_dir, "main.ts", &content)?; + // Remove any stale main.js so we never confuse a leftover (e.g. unbundled TS source + // a caller dropped at this path) with a fresh Bun bundle output. + let _ = std::fs::remove_file(&origin); build_loader( job_dir, base_internal_url, @@ -1170,11 +1200,25 @@ pub async fn prebundle_bun_script( ) .await?; + ensure_bundle_output_exists(&origin)?; + save_cache(&local_path, &remote_path, &origin, false).await?; Ok(()) } +/// Refuse to cache a bundle if `Bun.build` finished without producing the +/// expected output file. Belt-and-suspenders for any silent-failure mode the +/// upstream wait-status / `result.success` checks don't already trip on. +pub fn ensure_bundle_output_exists(bundle_path: &str) -> Result<()> { + if !std::path::Path::new(bundle_path).exists() { + return Err(error::Error::ExecutionErr(format!( + "bun bundle output missing at {bundle_path} after Bun.build — refusing to cache" + ))); + } + Ok(()) +} + pub const BUN_BUNDLE_OBJECT_STORE_PREFIX: &str = "bun_bundle/"; async fn get_script_import_updated_at(db: &DB, w_id: &str, script_path: &str) -> Result { @@ -1902,15 +1946,10 @@ try {{ &mut Some(occupancy_metrics), ) .await?; + let bundle_path = format!("{job_dir}/main.js"); + ensure_bundle_output_exists(&bundle_path)?; if !local_path.is_empty() { - match save_cache( - &local_path, - &remote_path, - &format!("{job_dir}/main.js"), - false, - ) - .await - { + match save_cache(&local_path, &remote_path, &bundle_path, false).await { Err(e) => { let em = format!("could not save {local_path} to bundle cache: {e:?}"); tracing::error!(em) diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 2634824d5c..0d5df81a83 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -90,9 +90,10 @@ pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZ pub use worker::*; pub use bun_executor::{ - build_loader, compute_bundle_local_and_remote_path, get_common_bun_proc_envs, - install_bun_lockfile, prebundle_bun_script, prepare_job_dir, LoaderMode, - BUN_DEDICATED_WORKER_ARGS, RELATIVE_BUN_BUILDER, RELATIVE_BUN_LOADER, + build_loader, compute_bundle_local_and_remote_path, ensure_bundle_output_exists, + generate_bun_bundle, get_common_bun_proc_envs, install_bun_lockfile, prebundle_bun_script, + prepare_job_dir, LoaderMode, BUN_DEDICATED_WORKER_ARGS, RELATIVE_BUN_BUILDER, + RELATIVE_BUN_LOADER, }; #[cfg(any(feature = "private", test))] pub use bun_executor::{ From 73358c29a49e8505748e0aee551164d31b64d031 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Wed, 6 May 2026 12:25:50 +0200 Subject: [PATCH 37/56] fix: autofocus searchbar and open dropdown on typing (#9052) * Autofocus searchbar and open dropdown on typing * nit always call onKeyDown --- frontend/src/lib/components/FilterSearchbar.svelte | 6 +++++- frontend/src/lib/components/RunsPage.svelte | 1 + frontend/src/lib/components/TaggedTextInput.svelte | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index 661b84cc87..537e76ceff 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -263,6 +263,7 @@ presets?: { name: string; value: string }[] class?: string placeholder?: string + autofocus?: boolean } type SchemaT = FilterSchemaRec // TODO: Generic @@ -271,7 +272,8 @@ value: valueInput = $bindable(), presets: _presets = [], class: className, - placeholder = 'Filter...' + placeholder = 'Filter...', + autofocus }: Props = $props() let _value = new DebouncedTempValue( @@ -604,6 +606,8 @@ inputSizeClasses.md )} {placeholder} + onKeyDown={() => (open = true)} + {autofocus} /> {#if asText.val} (_value.current = {})} /> diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 588b4cdaa4..86b7588099 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -755,6 +755,7 @@ })} bind:value={filters.val} placeholder="Filter runs..." + autofocus />
diff --git a/frontend/src/lib/components/TaggedTextInput.svelte b/frontend/src/lib/components/TaggedTextInput.svelte index ecf4904711..23bbc65e6b 100644 --- a/frontend/src/lib/components/TaggedTextInput.svelte +++ b/frontend/src/lib/components/TaggedTextInput.svelte @@ -6,6 +6,8 @@ highlights, onCurrentTagChange, onTextSegmentAtCursorChange, + onKeyDown, + autofocus, class: className = '' }: { tags: { regex: RegExp; id: string; onClear?: () => void }[] @@ -14,6 +16,8 @@ highlights?: { regex: RegExp; classes: string }[] onCurrentTagChange?: (tag: { id: string } | null) => void onTextSegmentAtCursorChange?: (segment: { text: string; start: number; end: number }) => void + onKeyDown?: (e: KeyboardEvent) => void + autofocus?: boolean class?: string } = $props() @@ -332,6 +336,7 @@ } function handleKeyDown(e: KeyboardEvent) { + onKeyDown?.(e) if (e.key === 'ArrowDown' || e.key === 'ArrowUp' || e.key === 'Enter') return const cursorPos = getCursorPosition() const text = getTextContent() @@ -501,6 +506,7 @@ } +