feat: Add possibility to delete flow step results when the flow is complete (#2806)

* feat: Add possibility to delete flow step results when the flow is complete

* Add third layer of tabs and gate feature to EE

backend/windmill-common/src/flow_status.rs

@@ -29,6 +29,7 @@ pub struct FlowStatus {
     pub step: i32,
     pub modules: Vec<FlowStatusModule>,
     pub failure_module: FlowStatusModuleWParent,
+    pub cleanup_module: FlowCleanupModule,
     #[serde(default)]
     #[serde(skip_serializing_if = "is_retry_default")]
     pub retry: RetryStatus,
@@ -96,6 +97,15 @@ pub struct FlowStatusModuleWParent {
     pub module_status: FlowStatusModule,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct FlowCleanupModule {
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub flow_jobs_to_clean: Vec<Uuid>,
+    #[serde(flatten)]
+    pub module_status: FlowStatusModule,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(tag = "type")]
 pub enum FlowStatusModule {
@@ -218,6 +228,16 @@ impl FlowStatus {
                         .unwrap_or_else(|| "failure".to_string()),
                 },
             },
+            cleanup_module: FlowCleanupModule {
+                flow_jobs_to_clean: vec![],
+                module_status: FlowStatusModule::WaitingForPriorSteps {
+                    id: f
+                        .failure_module
+                        .as_ref()
+                        .map(|x| x.id.clone())
+                        .unwrap_or_else(|| "cleanup".to_string()),
+                },
+            },
             retry: RetryStatus { fail_count: 0, failed_jobs: vec![] },
             restarted_from: None,
         }

backend/windmill-common/src/flows.rs

@@ -220,6 +220,8 @@ pub struct FlowModule {
     #[serde(skip_serializing_if = "Option::is_none")]
     // Priority at the flow step level
     pub priority: Option<i16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub delete_after_use: Option<bool>,
 }
 
 impl FlowModule {
@@ -369,6 +371,7 @@ pub fn add_virtual_items_if_necessary(modules: &mut Vec<FlowModule>) {
             cache_ttl: None,
             timeout: None,
             priority: None,
+            delete_after_use: None,
         });
     }
 }

backend/windmill-common/src/jobs.rs

@@ -335,9 +335,13 @@ pub async fn script_path_to_payload(
     script_path: &str,
     db: &DB,
     w_id: &str,
-) -> error::Result<(JobPayload, Option<Tag>)> {
-    let (job_payload, tag) = if script_path.starts_with("hub/") {
-        (JobPayload::ScriptHub { path: script_path.to_owned() }, None)
+) -> error::Result<(JobPayload, Option<Tag>, Option<bool>)> {
+    let (job_payload, tag, delete_after_use) = if script_path.starts_with("hub/") {
+        (
+            JobPayload::ScriptHub { path: script_path.to_owned() },
+            None,
+            None,
+        )
     } else {
         let (
             script_hash,
@@ -348,6 +352,7 @@ pub async fn script_path_to_payload(
             language,
             dedicated_worker,
             priority,
+            delete_after_use,
         ) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
         (
             JobPayload::ScriptHash {
@@ -361,9 +366,10 @@ pub async fn script_path_to_payload(
                 priority,
             },
             tag,
+            delete_after_use,
         )
     };
-    Ok((job_payload, tag))
+    Ok((job_payload, tag, delete_after_use))
 }
 
 pub async fn script_hash_to_tag_and_limits<'c>(
@@ -378,9 +384,10 @@ pub async fn script_hash_to_tag_and_limits<'c>(
     ScriptLang,
     Option<bool>,
     Option<i16>,
+    Option<bool>,
 )> {
     let script = sqlx::query!(
-        "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority from script where hash = $1 AND workspace_id = $2",
+        "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where hash = $1 AND workspace_id = $2",
         script_hash.0,
         w_id
     )
@@ -399,6 +406,7 @@ pub async fn script_hash_to_tag_and_limits<'c>(
         script.language,
         script.dedicated_worker,
         script.priority,
+        script.delete_after_use,
     ))
 }
 
@@ -407,7 +415,7 @@ pub async fn get_payload_tag_from_prefixed_path(
     db: &DB,
     w_id: &str,
 ) -> Result<(JobPayload, Option<String>), Error> {
-    let (payload, tag) = if path.starts_with("script/") {
+    let (payload, tag, _) = if path.starts_with("script/") {
         script_path_to_payload(path.strip_prefix("script/").unwrap(), &db, w_id).await?
     } else if path.starts_with("flow/") {
         let path = path.strip_prefix("flow/").unwrap().to_string();
@@ -421,7 +429,7 @@ pub async fn get_payload_tag_from_prefixed_path(
         let (tag, dedicated_worker) = r
             .map(|x| (x.tag, x.dedicated_worker))
             .unwrap_or_else(|| (None, None));
-        (JobPayload::Flow { path, dedicated_worker }, tag)
+        (JobPayload::Flow { path, dedicated_worker }, tag, None)
     } else {
         return Err(Error::BadRequest(format!(
             "path must start with script/ or flow/ (got {})",

backend/windmill-common/src/lib.rs

@@ -228,9 +228,10 @@ pub async fn get_latest_deployed_hash_for_path(
     ScriptLang,
     Option<bool>,
     Option<i16>,
+    Option<bool>,
 )> {
     let r_o = sqlx::query!(
-        "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority from script where path = $1 AND workspace_id = $2 AND
+        "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where path = $1 AND workspace_id = $2 AND
     created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
     deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
         script_path,
@@ -250,6 +251,7 @@ pub async fn get_latest_deployed_hash_for_path(
         script.language,
         script.dedicated_worker,
         script.priority,
+        script.delete_after_use,
     ))
 }