From 838266bb2fc2ea66f4aaacc0305c925f68208953 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 26 Jul 2023 21:14:05 +0200 Subject: [PATCH] feat: lock depedency for the entire flow + dependency job depend on script/flow's tag --- .github/workflows/docker-310-openbb.yml | 47 ++++++++++ .github/workflows/docker-310.yml | 13 ++- Dockerfile | 3 +- backend/windmill-api/src/flows.rs | 2 +- backend/windmill-api/src/scripts.rs | 2 +- backend/windmill-common/src/flows.rs | 1 - backend/windmill-queue/src/jobs.rs | 6 +- .../nsjail/run.bash.config.proto | 7 ++ backend/windmill-worker/src/worker.rs | 94 +++++++++++++------ backend/windmill-worker/src/worker_flow.rs | 1 - .../details/DetailPageDetailPanel.svelte | 29 +++++- .../details/DetailPageLayout.svelte | 3 +- .../(logged)/flows/get/[...path]/+page.svelte | 2 +- 13 files changed, 166 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/docker-310-openbb.yml diff --git a/.github/workflows/docker-310-openbb.yml b/.github/workflows/docker-310-openbb.yml new file mode 100644 index 0000000000..23b990217e --- /dev/null +++ b/.github/workflows/docker-310-openbb.yml @@ -0,0 +1,47 @@ +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +name: Build and push windmill with python 3.10 and openbb +on: workflow_dispatch + +concurrency: + group: ${{ github.ref }}-openbb + cancel-in-progress: true + +permissions: + contents: read + id-token: write + packages: write + +jobs: + build_ee: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + + - uses: depot/setup-action@v1 + + - name: Login to registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push publicly ee + uses: depot/build-push-action@v1 + with: + context: . + push: true + file: ./docker/DockerfileOpenbb + build-args: | + features=enterprise + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:openbb + labels: | + org.opencontainers.image.licenses=Windmill-Enterprise-License \ No newline at end of file diff --git a/.github/workflows/docker-310.yml b/.github/workflows/docker-310.yml index 23b990217e..e0df87334a 100644 --- a/.github/workflows/docker-310.yml +++ b/.github/workflows/docker-310.yml @@ -2,7 +2,7 @@ env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} -name: Build and push windmill with python 3.10 and openbb +name: Build and push windmill with python 3.10 on: workflow_dispatch concurrency: @@ -13,7 +13,7 @@ permissions: contents: read id-token: write packages: write - + jobs: build_ee: runs-on: ubuntu-22.04 @@ -33,15 +33,18 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Edit python version + run: | + sed -i 's/python:3.11.4/python:3.10.12/g' Dockerfile + - name: Build and push publicly ee uses: depot/build-push-action@v1 with: context: . push: true - file: ./docker/DockerfileOpenbb build-args: | features=enterprise tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:openbb + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:python310 labels: | - org.opencontainers.image.licenses=Windmill-Enterprise-License \ No newline at end of file + org.opencontainers.image.licenses=Windmill-Enterprise-License diff --git a/Dockerfile b/Dockerfile index dba877f032..1956d08230 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,7 +101,8 @@ RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denol RUN unzip deno.zip && rm deno.zip -FROM python:3.11.4-slim-buster +FROM python:3.11.4-slim-bookworm + ARG TARGETPLATFORM ARG APP=/usr/src/app diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 9338258d14..da081fd793 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -263,7 +263,7 @@ async fn create_flow( false, None, true, - None, + nf.tag, ) .await?; diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index fdf43aebc8..134a4253be 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -511,7 +511,7 @@ async fn create_script( false, None, true, - None, + ns.tag, ) .await?; tx = new_tx; diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 13563bd63e..36790c4f12 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -282,7 +282,6 @@ pub enum FlowModuleValue { concurrency_time_window_s: Option, }, Identity, - Http, } fn ordered_map(value: &HashMap, serializer: S) -> Result diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 398b149b04..86b19ad507 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1313,9 +1313,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( let flow_status = raw_flow.as_ref().map(FlowStatus::new); - let tag = if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies { - "dependency".to_string() - } else if job_kind == JobKind::Script_Hub { + let tag = if job_kind == JobKind::Script_Hub { "hub".to_string() } else { if tag == Some("".to_string()) { @@ -1327,6 +1325,8 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( } else if job_kind == JobKind::Identity { // identity is a light script, nativets is too "nativets" + } else if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies { + "dependency" } else { "deno" } diff --git a/backend/windmill-worker/nsjail/run.bash.config.proto b/backend/windmill-worker/nsjail/run.bash.config.proto index 6399514a3a..525f88e481 100644 --- a/backend/windmill-worker/nsjail/run.bash.config.proto +++ b/backend/windmill-worker/nsjail/run.bash.config.proto @@ -13,6 +13,7 @@ clone_newuser: {CLONE_NEWUSER} keep_caps: false keep_env: true +mount_proc: true mount { src: "/bin" @@ -20,6 +21,12 @@ mount { is_bind: true } +mount { + src: "/opt/microsoft" + dst: "/opt/microsoft" + is_bind: true +} + mount { src: "/lib" dst: "/lib" diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index c62d3f02cd..e719eac3c8 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -26,7 +26,7 @@ use tracing::{trace_span, Instrument}; use uuid::Uuid; use windmill_common::{ error::{self, to_anyhow, Error}, - flows::{FlowModuleValue, FlowValue}, + flows::{FlowModuleValue, FlowValue, FlowModule}, scripts::{ScriptHash, ScriptLang, get_full_hub_script_by_path}, utils::{rd_string, StripPath}, variables, BASE_URL, users::SUPERADMIN_SECRET_EMAIL, METRICS_ENABLED, jobs::{JobKind, QueuedJob, Metrics}, IS_READY, @@ -2152,9 +2152,72 @@ async fn handle_flow_dependency_job( )) })?; let mut flow = serde_json::from_value::(raw_flow).map_err(to_anyhow)?; + + flow.modules = lock_modules(flow.modules, job, logs, job_dir, db, worker_name, worker_dir, job_path.clone()).await?; + let new_flow_value = serde_json::to_value(flow).map_err(to_anyhow)?; + + // Re-check cancelation to ensure we don't accidentially override a flow. + if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id) + .fetch_optional(db) + .await + .map(|v| Some(true) == v) + .unwrap_or_else(|err| { + tracing::error!(%job.id, %err, "error checking cancelation for job {0}: {err}", job.id); + false + }) + { + return Ok(()); + } + + sqlx::query!( + "UPDATE flow SET value = $1 WHERE path = $2 AND workspace_id = $3", + new_flow_value, + job_path, + job.workspace_id + ) + .execute(db) + .await?; + Ok(()) +} + +#[async_recursion] +async fn lock_modules( + modules: Vec, + job: &QueuedJob, + logs: &mut String, + job_dir: &str, + db: &sqlx::Pool, + worker_name: &str, + worker_dir: &str, + job_path: String) -> Result> { let mut new_flow_modules = Vec::new(); - for mut e in flow.modules.into_iter() { + for mut e in modules.into_iter() { let FlowModuleValue::RawScript { lock: _, path, content, language, input_transforms, tag, concurrent_limit, concurrency_time_window_s} = e.value else { + match e.value { + FlowModuleValue::ForloopFlow { iterator, modules, skip_failures, parallel } => { + e.value = FlowModuleValue::ForloopFlow { iterator, modules: lock_modules(modules, job, logs, job_dir, db, worker_name, worker_dir, job_path.clone()).await?, skip_failures, parallel } + }, + FlowModuleValue::BranchAll { branches, parallel } => { + let mut nbranches = vec![]; + for mut b in branches { + b.modules = lock_modules(b.modules, job, logs, job_dir, db, worker_name, worker_dir, job_path.clone()).await?; + nbranches.push(b) + } + e.value = FlowModuleValue::BranchAll { branches: nbranches, parallel } + }, + FlowModuleValue::BranchOne { branches, default } => { + let mut nbranches = vec![]; + for mut b in branches { + b.modules = lock_modules(b.modules, job, logs, job_dir, db, worker_name, worker_dir, job_path.clone()).await?; + nbranches.push(b) + } + let default = lock_modules(default, job, logs, job_dir, db, worker_name, worker_dir, job_path.clone()).await?; + e.value = FlowModuleValue::BranchOne { branches: nbranches, default}; + } + _ => { + () + } + }; new_flow_modules.push(e); continue; }; @@ -2214,33 +2277,8 @@ async fn handle_flow_dependency_job( } } } - flow.modules = new_flow_modules; - let new_flow_value = serde_json::to_value(flow).map_err(to_anyhow)?; - - // Re-check cancelation to ensure we don't accidentially override a flow. - if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id) - .fetch_optional(db) - .await - .map(|v| Some(true) == v) - .unwrap_or_else(|err| { - tracing::error!(%job.id, %err, "error checking cancelation for job {0}: {err}", job.id); - false - }) - { - return Ok(()); - } - - sqlx::query!( - "UPDATE flow SET value = $1 WHERE path = $2 AND workspace_id = $3", - new_flow_value, - job_path, - job.workspace_id - ) - .execute(db) - .await?; - Ok(()) + Ok(new_flow_modules) } - async fn capture_dependency_job( job_id: &Uuid, job_language: &ScriptLang, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 72a379c798..691249de51 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -1697,7 +1697,6 @@ async fn compute_next_flow_transform( }; match &module.value { FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity), - FlowModuleValue::Http => trivial_next_job(JobPayload::Http), FlowModuleValue::Flow { path, .. } => { let payload = JobPayload::Flow(path.to_string()); Ok(NextFlowTransform::Continue( diff --git a/frontend/src/lib/components/details/DetailPageDetailPanel.svelte b/frontend/src/lib/components/details/DetailPageDetailPanel.svelte index fe3f66d780..be539b3b32 100644 --- a/frontend/src/lib/components/details/DetailPageDetailPanel.svelte +++ b/frontend/src/lib/components/details/DetailPageDetailPanel.svelte @@ -1,9 +1,14 @@ @@ -15,6 +20,9 @@ {#if !isOperator} Details & Triggers {/if} + {#if flow_json} + JSON + {/if}
@@ -61,6 +69,25 @@ + +
+ + +
+
diff --git a/frontend/src/lib/components/details/DetailPageLayout.svelte b/frontend/src/lib/components/details/DetailPageLayout.svelte index 3ed3c34481..0bd00a2b44 100644 --- a/frontend/src/lib/components/details/DetailPageLayout.svelte +++ b/frontend/src/lib/components/details/DetailPageLayout.svelte @@ -5,6 +5,7 @@ import DetailPageDetailPanel from './DetailPageDetailPanel.svelte' export let isOperator: boolean = false + export let flow_json: any | undefined = undefined let mobileTab: 'form' | 'detail' = 'form' @@ -18,7 +19,7 @@ - + diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index c7f1a954d4..2e072bb1f2 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -208,7 +208,7 @@ /> {#if flow} - +