feat: lock depedency for the entire flow + dependency job depend on script/flow's tag

.github/workflows/docker-310-openbb.yml

@@ -0,0 +1,47 @@
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+name: Build and push windmill with python 3.10 and openbb
+on: workflow_dispatch
+
+concurrency:
+  group: ${{ github.ref }}-openbb
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  id-token: write
+  packages: write
+  
+jobs:
+  build_ee:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v2
+
+      - uses: depot/setup-action@v1
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push publicly ee
+        uses: depot/build-push-action@v1
+        with:
+          context: .
+          push: true
+          file: ./docker/DockerfileOpenbb
+          build-args: |
+            features=enterprise
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:openbb
+          labels: |
+            org.opencontainers.image.licenses=Windmill-Enterprise-License

.github/workflows/docker-310.yml

@@ -2,7 +2,7 @@ env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
-name: Build and push windmill with python 3.10 and openbb
+name: Build and push windmill with python 3.10
 on: workflow_dispatch
 
 concurrency:
@@ -13,7 +13,7 @@ permissions:
   contents: read
   id-token: write
   packages: write
-  
+
 jobs:
   build_ee:
     runs-on: ubuntu-22.04
@@ -33,15 +33,18 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Edit python version
+        run: |
+          sed -i 's/python:3.11.4/python:3.10.12/g' Dockerfile
+
       - name: Build and push publicly ee
         uses: depot/build-push-action@v1
         with:
           context: .
           push: true
-          file: ./docker/DockerfileOpenbb
           build-args: |
             features=enterprise
           tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:openbb
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:python310
           labels: |
-            org.opencontainers.image.licenses=Windmill-Enterprise-License
+            org.opencontainers.image.licenses=Windmill-Enterprise-License

Dockerfile

@@ -101,7 +101,8 @@ RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denol
 
 RUN unzip deno.zip && rm deno.zip
 
-FROM python:3.11.4-slim-buster
+FROM python:3.11.4-slim-bookworm
+
 ARG TARGETPLATFORM
 
 ARG APP=/usr/src/app

backend/windmill-api/src/flows.rs

@@ -263,7 +263,7 @@ async fn create_flow(
         false,
         None,
         true,
-        None,
+        nf.tag,
     )
     .await?;
 

backend/windmill-api/src/scripts.rs

@@ -511,7 +511,7 @@ async fn create_script(
             false,
             None,
             true,
-            None,
+            ns.tag,
         )
         .await?;
         tx = new_tx;

backend/windmill-common/src/flows.rs

@@ -282,7 +282,6 @@ pub enum FlowModuleValue {
         concurrency_time_window_s: Option<i32>,
     },
     Identity,
-    Http,
 }
 
 fn ordered_map<S>(value: &HashMap<String, InputTransform>, serializer: S) -> Result<S::Ok, S::Error>

backend/windmill-queue/src/jobs.rs

@@ -1313,9 +1313,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
 
     let flow_status = raw_flow.as_ref().map(FlowStatus::new);
 
-    let tag = if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies {
-        "dependency".to_string()
-    } else if job_kind == JobKind::Script_Hub {
+    let tag = if job_kind == JobKind::Script_Hub {
         "hub".to_string()
     } else {
         if tag == Some("".to_string()) {
@@ -1327,6 +1325,8 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             } else if job_kind == JobKind::Identity {
                 // identity is a light script, nativets is too
                 "nativets"
+            } else if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies {
+                "dependency"
             } else {
                 "deno"
             }

backend/windmill-worker/nsjail/run.bash.config.proto

@@ -13,6 +13,7 @@ clone_newuser: {CLONE_NEWUSER}
 
 keep_caps: false
 keep_env: true
+mount_proc: true
 
 mount {
     src: "/bin"
@@ -20,6 +21,12 @@ mount {
 	is_bind: true
 }
 
+mount {
+    src: "/opt/microsoft"
+    dst: "/opt/microsoft"
+	is_bind: true
+}
+
 mount {
     src: "/lib"
     dst: "/lib"

backend/windmill-worker/src/worker.rs

@@ -26,7 +26,7 @@ use tracing::{trace_span, Instrument};
 use uuid::Uuid;
 use windmill_common::{
     error::{self, to_anyhow, Error},
-    flows::{FlowModuleValue, FlowValue},
+    flows::{FlowModuleValue, FlowValue, FlowModule},
     scripts::{ScriptHash, ScriptLang, get_full_hub_script_by_path},
     utils::{rd_string, StripPath},
     variables, BASE_URL, users::SUPERADMIN_SECRET_EMAIL, METRICS_ENABLED, jobs::{JobKind, QueuedJob, Metrics}, IS_READY,
@@ -2152,9 +2152,72 @@ async fn handle_flow_dependency_job(
         ))
     })?;
     let mut flow = serde_json::from_value::<FlowValue>(raw_flow).map_err(to_anyhow)?;
+
+    flow.modules = lock_modules(flow.modules, job, logs, job_dir, db, worker_name, worker_dir, job_path.clone()).await?;
+    let new_flow_value = serde_json::to_value(flow).map_err(to_anyhow)?;
+
+    // Re-check cancelation to ensure we don't accidentially override a flow.
+    if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id)
+        .fetch_optional(db)
+        .await
+        .map(|v| Some(true) == v)
+        .unwrap_or_else(|err| {
+            tracing::error!(%job.id, %err, "error checking cancelation for job {0}: {err}", job.id);
+            false
+        })
+    {
+        return Ok(());
+    }
+
+    sqlx::query!(
+        "UPDATE flow SET value = $1 WHERE path = $2 AND workspace_id = $3",
+        new_flow_value,
+        job_path,
+        job.workspace_id
+    )
+    .execute(db)
+    .await?;
+    Ok(())
+}
+
+#[async_recursion]
+async fn lock_modules(
+    modules: Vec<FlowModule>,     
+    job: &QueuedJob,
+    logs: &mut String,
+    job_dir: &str,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    worker_name: &str,
+    worker_dir: &str,
+    job_path: String) -> Result<Vec<FlowModule>> {
     let mut new_flow_modules = Vec::new();
-    for mut e in flow.modules.into_iter() {
+    for mut e in modules.into_iter() {
         let FlowModuleValue::RawScript { lock: _, path, content, language, input_transforms, tag, concurrent_limit, concurrency_time_window_s} = e.value else {
+            match e.value {
+                FlowModuleValue::ForloopFlow { iterator, modules, skip_failures, parallel } => {
+                    e.value = FlowModuleValue::ForloopFlow { iterator, modules: lock_modules(modules, job, logs, job_dir, db, worker_name, worker_dir, job_path.clone()).await?, skip_failures, parallel }
+                },
+                FlowModuleValue::BranchAll { branches, parallel } => {
+                    let mut nbranches = vec![];
+                    for mut b in branches {
+                        b.modules = lock_modules(b.modules, job, logs, job_dir, db, worker_name, worker_dir, job_path.clone()).await?;
+                        nbranches.push(b)
+                    }
+                    e.value = FlowModuleValue::BranchAll { branches: nbranches, parallel }
+                },
+                FlowModuleValue::BranchOne { branches, default } => {
+                    let mut nbranches = vec![];
+                    for mut b in branches {
+                        b.modules = lock_modules(b.modules, job, logs, job_dir, db, worker_name, worker_dir, job_path.clone()).await?;
+                        nbranches.push(b)
+                    }
+                    let default = lock_modules(default, job, logs, job_dir, db, worker_name, worker_dir, job_path.clone()).await?;
+                    e.value = FlowModuleValue::BranchOne { branches: nbranches, default};
+                }
+                _ => {
+                    ()
+                }
+            };
             new_flow_modules.push(e);
             continue;
         };
@@ -2214,33 +2277,8 @@ async fn handle_flow_dependency_job(
             }
         }
     }
-    flow.modules = new_flow_modules;
-    let new_flow_value = serde_json::to_value(flow).map_err(to_anyhow)?;
-
-    // Re-check cancelation to ensure we don't accidentially override a flow.
-    if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id)
-        .fetch_optional(db)
-        .await
-        .map(|v| Some(true) == v)
-        .unwrap_or_else(|err| {
-            tracing::error!(%job.id, %err, "error checking cancelation for job {0}: {err}", job.id);
-            false
-        })
-    {
-        return Ok(());
-    }
-
-    sqlx::query!(
-        "UPDATE flow SET value = $1 WHERE path = $2 AND workspace_id = $3",
-        new_flow_value,
-        job_path,
-        job.workspace_id
-    )
-    .execute(db)
-    .await?;
-    Ok(())
+    Ok(new_flow_modules)
 }
-
 async fn capture_dependency_job(
     job_id: &Uuid,
     job_language: &ScriptLang,

backend/windmill-worker/src/worker_flow.rs

@@ -1697,7 +1697,6 @@ async fn compute_next_flow_transform(
     };
     match &module.value {
         FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity),
-        FlowModuleValue::Http => trivial_next_job(JobPayload::Http),
         FlowModuleValue::Flow { path, .. } => {
             let payload = JobPayload::Flow(path.to_string());
             Ok(NextFlowTransform::Continue(

frontend/src/lib/components/details/DetailPageDetailPanel.svelte

@@ -1,9 +1,14 @@
 <script lang="ts">
-	import { Tabs, Tab, TabContent } from '$lib/components/common'
+	import { Tabs, Tab, TabContent, Button } from '$lib/components/common'
+	import { copyToClipboard } from '$lib/utils'
+	import { faClipboard } from '@fortawesome/free-solid-svg-icons'
 	import { CalendarCheck2, Terminal, Webhook } from 'lucide-svelte'
+	import { Highlight } from 'svelte-highlight'
+	import json from 'svelte-highlight/languages/json'
 	import { Pane, Splitpanes } from 'svelte-splitpanes'
 
 	let triggerSelected: 'webhooks' | 'schedule' | 'cli' = 'webhooks'
+	export let flow_json: any | undefined = undefined
 
 	export let isOperator: boolean = false
 </script>
@@ -15,6 +20,9 @@
 			{#if !isOperator}
 				<Tab value="details">Details & Triggers</Tab>
 			{/if}
+			{#if flow_json}
+				<Tab value="flow_json">JSON</Tab>
+			{/if}
 			<svelte:fragment slot="content">
 				<div class="overflow-hidden" style="height:calc(100% - 32px);">
 					<TabContent value="saved_inputs" class="flex flex-col flex-1 h-full">
@@ -61,6 +69,25 @@
 							</Pane>
 						</Splitpanes>
 					</TabContent>
+					<TabContent value="flow_json" class="flex flex-col flex-1 h-full">
+						<div class="relative pt-2">
+							<Button
+								on:click={() => copyToClipboard(JSON.stringify(flow_json, null, 4))}
+								color="light"
+								variant="border"
+								size="xs"
+								startIcon={{ icon: faClipboard }}
+								btnClasses="absolute top-2 right-2 w-min"
+							>
+								Copy content
+							</Button>
+							<Highlight
+								language={json}
+								code={JSON.stringify(flow_json, null, 4)}
+								class="overflow-auto"
+							/>
+						</div>
+					</TabContent>
 				</div>
 			</svelte:fragment>
 		</Tabs>

frontend/src/lib/components/details/DetailPageLayout.svelte

@@ -5,6 +5,7 @@
 	import DetailPageDetailPanel from './DetailPageDetailPanel.svelte'
 
 	export let isOperator: boolean = false
+	export let flow_json: any | undefined = undefined
 
 	let mobileTab: 'form' | 'detail' = 'form'
 </script>
@@ -18,7 +19,7 @@
 					<slot name="form" />
 				</Pane>
 				<Pane size={35} minSize={15}>
-					<DetailPageDetailPanel {isOperator}>
+					<DetailPageDetailPanel {isOperator} {flow_json}>
 						<slot slot="webhooks" name="webhooks" />
 						<slot slot="schedule" name="schedule" />
 						<slot slot="cli" name="cli" />

frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte

@@ -208,7 +208,7 @@
 />
 
 {#if flow}
-	<DetailPageLayout isOperator={$userStore?.operator}>
+	<DetailPageLayout isOperator={$userStore?.operator} flow_json={flow.value}>
 		<svelte:fragment slot="header">
 			<DetailPageHeader
 				{mainButtons}