From fbcac1e0a6eb4e51e991e76f9b30a3de6fdeca39 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 6 Aug 2024 01:02:30 +0200 Subject: [PATCH 01/13] support tarbundle in vscode extension --- backend/windmill-api/src/jobs.rs | 59 ++++++++++++++------- backend/windmill-common/src/scripts.rs | 1 + backend/windmill-worker/src/global_cache.rs | 2 +- backend/windmill-worker/src/worker.rs | 37 +++++++------ cli/script.ts | 2 +- frontend/src/lib/components/Dev.svelte | 20 +++++-- 6 files changed, 77 insertions(+), 44 deletions(-) diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 622a1bb0ad..f5ece1b195 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -184,7 +184,10 @@ pub fn workspaced_service() -> Router { .layer(ce_headers.clone()), ) .route("/run/preview", post(run_preview_script)) - .route("/run/preview_bundle", post(run_bundle_preview_script)) + .route( + "/run/preview_bundle", + post(run_bundle_preview_script).layer(axum::extract::DefaultBodyLimit::disable()), + ) .route("/add_batch_jobs/:n", post(add_batch_jobs)) .route("/run/preview_flow", post(run_preview_flow_job)) .route( @@ -2611,6 +2614,7 @@ enum PreviewKind { Http, Noop, Bundle, + Tarbundle, } #[derive(Deserialize)] @@ -3810,6 +3814,8 @@ async fn run_bundle_preview_script( Query(run_query): Query, mut multipart: axum::extract::Multipart, ) -> error::Result<(StatusCode, String)> { + use windmill_common::scripts::PREVIEW_IS_TAR_CODEBASE_HASH; + check_license_key_valid().await?; check_scopes(&authed, || format!("runscript"))?; @@ -3822,9 +3828,12 @@ async fn run_bundle_preview_script( let mut job_id = None; let mut tx = None; let mut uploaded = false; + let mut is_tar = false; + while let Some(field) = multipart.next_field().await.unwrap() { let name = field.name().unwrap().to_string(); - let data = field.bytes().await.unwrap(); + let data = field.bytes().await; + let data = data.map_err(to_anyhow)?; if name == "preview" { let preview: Preview = serde_json::from_slice(&data).map_err(to_anyhow)?; @@ -3836,27 +3845,33 @@ async fn run_bundle_preview_script( let args = preview.args.unwrap_or_default(); + is_tar = match preview.kind { + Some(PreviewKind::Tarbundle) => true, + _ => false, + }; + + // tracing::info!("is_tar 1: {is_tar}"); // hmap.insert("") let (uuid, ntx) = push( &db, ltx, &w_id, - match preview.kind { - Some(PreviewKind::Identity) => JobPayload::Identity, - Some(PreviewKind::Noop) => JobPayload::Noop, - _ => JobPayload::Code(RawCode { - hash: Some(PREVIEW_IS_CODEBASE_HASH), - content: preview.content.unwrap_or_default(), - path: preview.path, - language: preview.language.unwrap_or(ScriptLang::Deno), - lock: preview.lock, - concurrent_limit: None, // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here - concurrency_time_window_s: None, // TODO(gbouv): same as above - cache_ttl: None, - dedicated_worker: preview.dedicated_worker, - custom_concurrency_key: None, - }), - }, + JobPayload::Code(RawCode { + hash: if is_tar { + Some(PREVIEW_IS_TAR_CODEBASE_HASH) + } else { + Some(PREVIEW_IS_CODEBASE_HASH) + }, + content: preview.content.unwrap_or_default(), + path: preview.path, + language: preview.language.unwrap_or(ScriptLang::Deno), + lock: preview.lock, + concurrent_limit: None, // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here + concurrency_time_window_s: None, // TODO(gbouv): same as above + cache_ttl: None, + dedicated_worker: preview.dedicated_worker, + custom_concurrency_key: None, + }), PushArgs::from(&args), authed.display_username(), &authed.email, @@ -3881,7 +3896,7 @@ async fn run_bundle_preview_script( tx = Some(ntx); } if name == "file" { - let id = job_id + let mut id = job_id .as_ref() .ok_or_else(|| { Error::BadRequest( @@ -3890,6 +3905,12 @@ async fn run_bundle_preview_script( })? .to_string(); + // tracing::info!("is_tar 2: {is_tar}"); + + if is_tar { + id = format!("{}.tar", id); + } + uploaded = true; if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index ac49ca3746..86c506f467 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -133,6 +133,7 @@ impl Display for ScriptKind { } pub const PREVIEW_IS_CODEBASE_HASH: i64 = -42; +pub const PREVIEW_IS_TAR_CODEBASE_HASH: i64 = -43; #[derive(Serialize, sqlx::FromRow)] pub struct Script { diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 2abce202cb..12584b905d 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -111,7 +111,7 @@ pub async fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> { tracing::info!("Failed to untar to {folder}. Error: {:?}", e); fs::remove_dir_all(&folder).await?; return Err(error::Error::ExecutionErr(format!( - "Failed to untar piptar {folder}" + "Failed to untar tar {folder}" ))); } tracing::info!( diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 96bc14417a..e274f1790d 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -7,8 +7,7 @@ */ use windmill_common::{ - auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET}, - worker::{get_windmill_memory_usage, get_worker_memory_usage, TMP_DIR}, + auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET}, scripts::PREVIEW_IS_TAR_CODEBASE_HASH, worker::{get_windmill_memory_usage, get_worker_memory_usage, TMP_DIR} }; use anyhow::{Context, Result}; @@ -2792,23 +2791,23 @@ async fn handle_code_execution_job( envs, codebase, } = match job.job_kind { - JobKind::Preview => ContentReqLangEnvs { - content: job - .raw_code - .clone() - .unwrap_or_else(|| "no raw code".to_owned()), - lockfile: job.raw_lock.clone(), - language: job.language.to_owned(), - envs: None, - codebase: if job - .script_hash - .is_some_and(|y| y.0 == PREVIEW_IS_CODEBASE_HASH) - { - Some(job.id.to_string()) - } else { - None - }, - }, + JobKind::Preview => { + let codebase = match job.script_hash.map(|x| x.0) { + Some(PREVIEW_IS_CODEBASE_HASH) => Some(job.id.to_string()), + Some(PREVIEW_IS_TAR_CODEBASE_HASH) => Some(format!("{}.tar", job.id)), + _ => None, + }; + + ContentReqLangEnvs { + content: job + .raw_code + .clone() + .unwrap_or_else(|| "no raw code".to_owned()), + lockfile: job.raw_lock.clone(), + language: job.language.to_owned(), + envs: None, + codebase + }}, JobKind::Script_Hub => { get_hub_script_content_and_requirements(job.script_path.clone(), db).await? } diff --git a/cli/script.ts b/cli/script.ts index ba39bae292..4e130be7ea 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -47,7 +47,7 @@ import { } from "./conf.ts"; import { SyncCodebase, listSyncCodebases } from "./codebase.ts"; import fs from "node:fs"; -import { Tarball } from "npm:@ayonli/jsext/archive"; +import { type Tarball } from "npm:@ayonli/jsext/archive"; export interface ScriptFile { parent_hash?: string; diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index 4d211f6eb2..55a6bb80a6 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -196,11 +196,12 @@ replaceScript(event.data) } else if (event.data.type == 'testBundle') { if (event.data.id == lastBundleCommandId) { - testBundle(event.data.file) + testBundle(event.data.file, event.data.isTar) } else { sendUserToast(`Bundle received ${lastBundleCommandId} was obsolete, ignoring`, true) } } else if (event.data.type == 'testBundleError') { + loadingCodebaseButton = false sendUserToast( typeof event.data.error == 'object' ? JSON.stringify(event.data.error) : event.data.error, true @@ -244,7 +245,7 @@ window.parent?.postMessage({ type: 'refresh' }, '*') }) - async function testBundle(file: string) { + async function testBundle(file: string, isTar: boolean) { testJobLoader?.abstractRun(async () => { try { const form = new FormData() @@ -252,14 +253,25 @@ 'preview', JSON.stringify({ content: currentScript?.content, - kind: 'bundle', + kind: isTar ? 'tarbundle' : 'bundle', path: currentScript?.path, args, language: currentScript?.language, tag: currentScript?.tag }) ) - form.append('file', file) + // sendUserToast(JSON.stringify(file)) + if (isTar) { + var array: number[] = [] + file = atob(file) + for (var i = 0; i < file.length; i++) { + array.push(file.charCodeAt(i)) + } + let blob = new Blob([new Uint8Array(array)], { type: 'application/octet-stream' }) + form.append('file', blob) + } else { + form.append('file', file) + } const url = '/api/w/' + workspace + '/jobs/run/preview_bundle' From 657f03bc67f84a8593b28483684c2ad8ad432865 Mon Sep 17 00:00:00 2001 From: Faton Ramadani Date: Tue, 6 Aug 2024 10:16:29 +0200 Subject: [PATCH 02/13] fix(frontend): fix the app created from a script or flow with the new topbar (#4194) --- .../components/details/createAppFromScript.ts | 294 +++++++++++++++++- 1 file changed, 290 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/components/details/createAppFromScript.ts b/frontend/src/lib/components/details/createAppFromScript.ts index 13054d4f00..52dd5d9caa 100644 --- a/frontend/src/lib/components/details/createAppFromScript.ts +++ b/frontend/src/lib/components/details/createAppFromScript.ts @@ -5,7 +5,7 @@ export function createAppFromScript(path: string, schema: Record | '3': { fixed: false, x: 0, - y: 0, + y: 2, w: 2, h: 8, fullHeight: false @@ -13,7 +13,7 @@ export function createAppFromScript(path: string, schema: Record | '12': { fixed: false, x: 0, - y: 0, + y: 2, w: 12, h: 21, fullHeight: false @@ -27,6 +27,38 @@ export function createAppFromScript(path: string, schema: Record | id: 'a' }, id: 'a' + }, + { + '3': { + fixed: false, + x: 0, + y: 8, + fullHeight: false, + w: 6, + h: 2 + }, + '12': { + fixed: false, + x: 0, + y: 0, + fullHeight: false, + w: 12, + h: 2 + }, + data: { + type: 'containercomponent', + configuration: {}, + customCss: { + container: { + class: '!p-0', + style: '' + } + }, + actions: [], + numberOfSubgrids: 1, + id: 'g' + }, + id: 'g' } ], fullscreen: false, @@ -34,6 +66,7 @@ export function createAppFromScript(path: string, schema: Record | hiddenInlineScripts: [], css: {}, norefreshbar: false, + hideLegacyTopBar: true, subgrids: { 'a-0': [ { @@ -376,6 +409,116 @@ export function createAppFromScript(path: string, schema: Record | }, id: 'f' } + ], + 'g-0': [ + { + '3': { + fixed: false, + x: 0, + y: 0, + fullHeight: false, + w: 6, + h: 1 + }, + '12': { + fixed: false, + x: 0, + y: 0, + fullHeight: false, + w: 6, + h: 1 + }, + data: { + type: 'textcomponent', + configuration: { + style: { + type: 'static', + value: 'Body' + }, + copyButton: { + type: 'static', + value: false + }, + tooltip: { + type: 'evalv2', + value: '', + fieldType: 'text', + expr: '`Author: ${ctx.author}`', + connections: [ + { + componentId: 'ctx', + id: 'author' + } + ] + }, + disableNoText: { + type: 'static', + value: true, + fieldType: 'boolean' + } + }, + componentInput: { + type: 'templatev2', + fieldType: 'template', + eval: '${ctx.summary}', + connections: [ + { + id: 'summary', + componentId: 'ctx' + } + ] + }, + customCss: { + text: { + class: 'text-xl font-semibold whitespace-nowrap truncate', + style: '' + }, + container: { + class: '', + style: '' + } + }, + actions: [], + horizontalAlignment: 'left', + verticalAlignment: 'center', + id: 'h' + }, + id: 'h' + }, + { + '3': { + fixed: false, + x: 0, + y: 1, + fullHeight: false, + w: 3, + h: 1 + }, + '12': { + fixed: false, + x: 6, + y: 0, + fullHeight: false, + w: 6, + h: 1 + }, + data: { + type: 'recomputeallcomponent', + configuration: {}, + customCss: { + container: { + style: '', + class: '' + } + }, + actions: [], + menuItems: [], + horizontalAlignment: 'right', + verticalAlignment: 'center', + id: 'i' + }, + id: 'i' + } ] } } @@ -424,7 +567,7 @@ export function createAppFromFlow(path: string, schema: Record | un '3': { fixed: false, x: 0, - y: 0, + y: 2, w: 2, h: 8, fullHeight: false @@ -432,7 +575,7 @@ export function createAppFromFlow(path: string, schema: Record | un '12': { fixed: false, x: 0, - y: 0, + y: 2, w: 12, h: 21, fullHeight: false @@ -446,6 +589,38 @@ export function createAppFromFlow(path: string, schema: Record | un id: 'a' }, id: 'a' + }, + { + '3': { + fixed: false, + x: 0, + y: 8, + fullHeight: false, + w: 6, + h: 2 + }, + '12': { + fixed: false, + x: 0, + y: 0, + fullHeight: false, + w: 12, + h: 2 + }, + data: { + type: 'containercomponent', + configuration: {}, + customCss: { + container: { + class: '!p-0', + style: '' + } + }, + actions: [], + numberOfSubgrids: 1, + id: 'g' + }, + id: 'g' } ], fullscreen: false, @@ -453,6 +628,7 @@ export function createAppFromFlow(path: string, schema: Record | un hiddenInlineScripts: [], css: {}, norefreshbar: false, + hideLegacyTopBar: true, subgrids: { 'a-0': [ { @@ -796,6 +972,116 @@ export function createAppFromFlow(path: string, schema: Record | un }, id: 'f' } + ], + 'g-0': [ + { + '3': { + fixed: false, + x: 0, + y: 0, + fullHeight: false, + w: 6, + h: 1 + }, + '12': { + fixed: false, + x: 0, + y: 0, + fullHeight: false, + w: 6, + h: 1 + }, + data: { + type: 'textcomponent', + configuration: { + style: { + type: 'static', + value: 'Body' + }, + copyButton: { + type: 'static', + value: false + }, + tooltip: { + type: 'evalv2', + value: '', + fieldType: 'text', + expr: '`Author: ${ctx.author}`', + connections: [ + { + componentId: 'ctx', + id: 'author' + } + ] + }, + disableNoText: { + type: 'static', + value: true, + fieldType: 'boolean' + } + }, + componentInput: { + type: 'templatev2', + fieldType: 'template', + eval: '${ctx.summary}', + connections: [ + { + id: 'summary', + componentId: 'ctx' + } + ] + }, + customCss: { + text: { + class: 'text-xl font-semibold whitespace-nowrap truncate', + style: '' + }, + container: { + class: '', + style: '' + } + }, + actions: [], + horizontalAlignment: 'left', + verticalAlignment: 'center', + id: 'h' + }, + id: 'h' + }, + { + '3': { + fixed: false, + x: 0, + y: 1, + fullHeight: false, + w: 3, + h: 1 + }, + '12': { + fixed: false, + x: 6, + y: 0, + fullHeight: false, + w: 6, + h: 1 + }, + data: { + type: 'recomputeallcomponent', + configuration: {}, + customCss: { + container: { + style: '', + class: '' + } + }, + actions: [], + menuItems: [], + horizontalAlignment: 'right', + verticalAlignment: 'center', + id: 'i' + }, + id: 'i' + } ] } } From 7b3128171ea7193efea569297a099bb9ee6935e1 Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Tue, 6 Aug 2024 10:31:08 +0200 Subject: [PATCH 03/13] feat: Tag filter on Runs page (#4193) * Add warning of job search parse error + front fixes * Add filter tag and add filters to small screen * Fill missing property --- .../src/lib/components/SavedInputs.svelte | 1 + .../src/lib/components/runs/JobLoader.svelte | 4 + .../src/lib/components/runs/RunsFilter.svelte | 159 +++++++++++++++++- .../search/GlobalSearchModal.svelte | 33 +++- .../components/search/QuickMenuItem.svelte | 8 +- .../(logged)/runs/[...path]/+page.svelte | 34 +++- 6 files changed, 229 insertions(+), 10 deletions(-) diff --git a/frontend/src/lib/components/SavedInputs.svelte b/frontend/src/lib/components/SavedInputs.svelte index 4aa1a7dad4..6275e95917 100644 --- a/frontend/src/lib/components/SavedInputs.svelte +++ b/frontend/src/lib/components/SavedInputs.svelte @@ -186,6 +186,7 @@ label={null} folder={null} concurrencyKey={null} + tag={null} success="running" argFilter={undefined} bind:loading diff --git a/frontend/src/lib/components/runs/JobLoader.svelte b/frontend/src/lib/components/runs/JobLoader.svelte index dc1cf7d228..463f1ca7b7 100644 --- a/frontend/src/lib/components/runs/JobLoader.svelte +++ b/frontend/src/lib/components/runs/JobLoader.svelte @@ -36,6 +36,7 @@ export let completedJobs: CompletedJob[] | undefined = undefined export let externalJobs: Job[] | undefined = undefined export let concurrencyKey: string | null + export let tag: string | null export let extendedJobs: ExtendedJobs | undefined = undefined export let argError = '' export let resultError = '' @@ -58,6 +59,7 @@ isSkipped != undefined && jobKinds && concurrencyKey && + tag && lookback && user && folder && @@ -143,6 +145,7 @@ ? true : undefined, label: label === null || label === '' ? undefined : label, + tag: tag === null || tag === '' ? undefined : tag, isNotSchedule: showSchedules == false ? true : undefined, scheduledForBeforeNow: showFutureJobs == false ? true : undefined, args: @@ -190,6 +193,7 @@ isSkipped: isSkipped ? undefined : false, isFlowStep: jobKindsCat != 'all' ? false : undefined, label: label === null || label === '' ? undefined : label, + tag: tag === null || tag === '' ? undefined : tag, isNotSchedule: showSchedules == false ? true : undefined, scheduledForBeforeNow: showFutureJobs == false ? true : undefined, args: diff --git a/frontend/src/lib/components/runs/RunsFilter.svelte b/frontend/src/lib/components/runs/RunsFilter.svelte index cca0369f7b..095467b69c 100644 --- a/frontend/src/lib/components/runs/RunsFilter.svelte +++ b/frontend/src/lib/components/runs/RunsFilter.svelte @@ -18,6 +18,7 @@ export let path: string | null = null export let label: string | null = null export let concurrencyKey: string | null = null + export let tag: string | null = null export let success: 'running' | 'success' | 'failure' | undefined = undefined export let isSkipped: boolean | undefined = undefined export let argFilter: string @@ -37,11 +38,12 @@ $: displayedLabel = label $: displayedConcurrencyKey = concurrencyKey + $: displayedTag = tag let copyArgFilter = argFilter let copyResultFilter = resultFilter - export let filterBy: 'path' | 'user' | 'folder' | 'label' | 'concurrencyKey' = 'path' + export let filterBy: 'path' | 'user' | 'folder' | 'label' | 'concurrencyKey' | 'tag' = 'path' const dispatch = createEventDispatcher() @@ -63,11 +65,15 @@ } else if (concurrencyKey !== null && concurrencyKey !== '' && filterBy !== 'concurrencyKey') { manuallySet = true filterBy = 'concurrencyKey' + } else if (tag !== null && tag !== '' && filterBy !== 'tag') { + manuallySet = true + filterBy = 'tag' } } let labelTimeout: NodeJS.Timeout | undefined = undefined let concurrencyKeyTimeout: NodeJS.Timeout | undefined = undefined + let tagTimeout: NodeJS.Timeout | undefined = undefined
@@ -94,6 +100,7 @@ folder = null label = null concurrencyKey = null + tag = null } else { manuallySet = false } @@ -105,7 +112,8 @@ @@ -295,6 +303,39 @@ />
{/key} + {:else if filterBy === 'tag'} + {#key tag} +
+ {#if tag} + + {/if} + Tag + + { + if (tagTimeout) { + clearTimeout(tagTimeout) + } + + tagTimeout = setTimeout(() => { + tag = displayedTag + }, 1000) + }} + /> +
+ {/key} {/if}
@@ -383,6 +424,8 @@ user = null folder = null label = null + concurrencyKey = null + tag = null } else { manuallySet = false } @@ -391,6 +434,9 @@ + + + @@ -415,10 +461,10 @@ items={usernames} value={user} bind:selectedItem={user} - inputClassName="!h-[32px] py-1 !text-xs !w-64" + inputClassName="!h-[32px] py-1 !text-xs !w-80" hideArrow className={user ? '!font-bold' : ''} - dropdownClassName="!font-normal !w-64 !max-w-64" + dropdownClassName="!font-normal !w-80 !max-w-80" />
@@ -445,10 +491,10 @@ items={folders} value={folder} bind:selectedItem={folder} - inputClassName="!h-[32px] py-1 !text-xs !w-64" + inputClassName="!h-[32px] py-1 !text-xs !w-80" hideArrow className={folder ? '!font-bold' : ''} - dropdownClassName="!font-normal !w-64 !max-w-64" + dropdownClassName="!font-normal !w-80 !max-w-80" /> @@ -483,6 +529,107 @@ {/key} + {:else if filterBy === 'tag'} + {#key tag} + + {/key} + {:else if filterBy === 'label'} + {#key label} + + {/key} + {:else if filterBy === 'concurrencyKey'} + {#key concurrencyKey} + + {/key} {/if} + {#if queryParseErrors.length > 0} + + + + Some of your search terms have been ignored because one or more parse errors:

+
    + {#each queryParseErrors as msg} +
  • - {msg}
    • + {/each} +
+ + + {/if}
{#if tab === 'default' || tab === 'switch-mode'} {@const items = (itemMap[tab] ?? []).filter((e) => defaultMenuItems.includes(e))} {#if items.length > 0} -
+
{#each items as el} {/each}
@@ -549,6 +578,7 @@ el.path + (el.starred ? ' ★' : '')} icon={iconForWindmillItem(el.type)} + bind:mouseMoved /> {/each} {/if} @@ -600,6 +630,7 @@ hovered={selectedItem && r?.document.id[0] === selectedItem?.document.id[0]} icon={r?.icon} containerClass="rounded-md px-2 py-1 my-2" + bind:mouseMoved >
dispatch('hover')} + on:mouseenter={() => { + if (mouseMoved) { + dispatch('hover') + } + mouseMoved=false + }} class={twMerge( `rounded-md w-full transition-all cursor-pointer ${ hovered ? 'bg-surface-hover' : '' diff --git a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte index 6078c7d4ad..864ecd8fe9 100644 --- a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte @@ -49,6 +49,7 @@ let folder: string | null = $page.url.searchParams.get('folder') let label: string | null = $page.url.searchParams.get('label') let concurrencyKey: string | null = $page.url.searchParams.get('concurrency_key') + let tag: string | null = $page.url.searchParams.get('tag') // Rest of filters handled by RunsFilter let success: 'running' | 'success' | 'failure' | undefined = ($page.url.searchParams.get( 'success' @@ -125,6 +126,7 @@ schedulePath || jobKindsCat || concurrencyKey || + tag || graph || minTs || maxTs || @@ -218,6 +220,12 @@ searchParams.delete('concurrency_key') } + if (tag) { + searchParams.set('tag', tag) + } else { + searchParams.delete('tag') + } + if (label) { searchParams.set('label', label) } else { @@ -287,6 +295,7 @@ folder = null label = null concurrencyKey = null + tag = null } function filterByUser(e: CustomEvent) { @@ -295,6 +304,7 @@ user = e.detail label = null concurrencyKey = null + tag = null } function filterByFolder(e: CustomEvent) { @@ -303,6 +313,7 @@ folder = e.detail label = null concurrencyKey = null + tag = null } function filterByLabel(e: CustomEvent) { @@ -311,6 +322,7 @@ folder = null label = e.detail concurrencyKey = null + tag = null } function filterByConcurrencyKey(e: CustomEvent) { @@ -319,6 +331,16 @@ folder = null label = null concurrencyKey = e.detail + tag = null + } + + function filterByTag(e: CustomEvent) { + path = null + user = null + folder = null + label = null + concurrencyKey = null + tag = e.detail } let calendarChangeTimeout: NodeJS.Timeout | undefined = undefined @@ -369,7 +391,8 @@ ? resultFilter : undefined, allWorkspaces: allWorkspaces ? true : undefined, - concurrencyKey: concurrencyKey ?? undefined + concurrencyKey: concurrencyKey ?? undefined, + tag: tag ?? undefined } selectedFiltersString = JSON.stringify(selectedFilters, null, 4) @@ -395,7 +418,7 @@ } const warnJobLimitMsg = - 'The exact number of concurrent job at the beginning of the time range may be incorrect as only the last 1000 jobs are taken into account: a job that was started earlier than this limit will not be taken into account' + 'The exact number of concurrent jobs at the beginning of the time range may be incorrect as only the last 1000 jobs are taken into account: a job that was started earlier than this limit will not be taken into account' $: warnJobLimit = graph === 'ConcurrencyChart' && @@ -430,6 +453,7 @@ {concurrencyKey} {argError} {resultError} + {tag} bind:loading bind:this={jobLoader} lookback={graphIsRunsChart ? 0 : lookback} @@ -521,6 +545,7 @@ bind:folder bind:label bind:concurrencyKey + bind:tag bind:path bind:success bind:argFilter @@ -815,6 +840,7 @@ on:filterByFolder={filterByFolder} on:filterByLabel={filterByLabel} on:filterByConcurrencyKey={filterByConcurrencyKey} + on:filterByTag={filterByTag} /> {:else}
@@ -872,6 +898,9 @@ bind:folder bind:path bind:user + bind:label + bind:concurrencyKey + bind:tag bind:success bind:argFilter bind:resultFilter @@ -1158,6 +1187,7 @@ on:filterByFolder={filterByFolder} on:filterByLabel={filterByLabel} on:filterByConcurrencyKey={filterByConcurrencyKey} + on:filterByTag={filterByTag} />
From a1c40d7fd0933fda0386835d6db1cae244befd2a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 6 Aug 2024 10:39:13 +0200 Subject: [PATCH 04/13] add timeout for app editor log panel --- frontend/src/lib/components/apps/editor/AppEditor.svelte | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index 58650cf296..812805ae13 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -536,6 +536,7 @@ } let runnableJobEnterTimeout: NodeJS.Timeout | undefined = undefined + let stillInJobEnter = false @@ -679,9 +680,15 @@ class="relative h-full w-full overflow-x-visible" on:mouseenter={() => { runnableJobEnterTimeout && clearTimeout(runnableJobEnterTimeout) - $runnableJob.focused = true + stillInJobEnter = true + runnableJobEnterTimeout = setTimeout(() => { + if (stillInJobEnter) { + $runnableJob.focused = true + } + }, 200) }} on:mouseleave={() => { + stillInJobEnter = false runnableJobEnterTimeout = setTimeout( () => ($runnableJob.focused = false), 200 From 5104dba63140e2186dc1eed7f0bf4e14e357bf3d Mon Sep 17 00:00:00 2001 From: Faton Ramadani Date: Tue, 6 Aug 2024 10:48:08 +0200 Subject: [PATCH 05/13] fix(frontend): Fr/improve suspend drawer (#4189) * fix(frontend): add missing info about the cancel url in the suspend drawer * fix(frontend): add missing info about the cancel url in the suspend drawer --- .../src/lib/components/flows/content/SuspendDrawer.svelte | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/flows/content/SuspendDrawer.svelte b/frontend/src/lib/components/flows/content/SuspendDrawer.svelte index 2cd76806ad..d743feb918 100644 --- a/frontend/src/lib/components/flows/content/SuspendDrawer.svelte +++ b/frontend/src/lib/components/flows/content/SuspendDrawer.svelte @@ -32,7 +32,8 @@
A prompt is simply an approval step that can be self-approved. To do this, include the resume url in the returned payload of the step. The UX will automatically adapt and show the - prompt to the operator when running the flow. e.g: + prompt to the operator when running the flow. Additionally, adding the cancel url will also + render a cancel button, providing the operator with an option to cancel the step. e.g: TypeScript (Bun) TypeScript (Deno) @@ -49,6 +50,7 @@ export async function main() { return { resume: urls['resume'], + cancel: urls['cancel'], default_args: {}, // optional, see below enums: {} // optional, see below } @@ -65,6 +67,7 @@ export async function main() { return { resume: urls['resume'], + cancel: urls['cancel'], default_args: {}, // optional, see below enums: {} // optional, see below } @@ -80,6 +83,7 @@ def main(): urls = wmill.get_resume_urls() return { "resume": urls["resume"], + "cancel": urls["cancel"], "default_args": {}, # optional, see below "enums": {} # optional, see below } From b6ab184889ca52e733b32f78573b15ee44635d9a Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Tue, 6 Aug 2024 10:48:22 +0200 Subject: [PATCH 06/13] Add warning of job search parse error + front fixes (#4187) From 39dc6857eb5c19f5a69c2a8e259b922e2ad2290d Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Tue, 6 Aug 2024 10:48:53 +0200 Subject: [PATCH 07/13] Remove admin requirement to cancel job selection (#4188) --- backend/windmill-api/src/jobs.rs | 1 - frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index f5ece1b195..706f588c9c 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -1462,7 +1462,6 @@ async fn cancel_selection( Path(w_id): Path, Json(jobs): Json>, ) -> error::JsonResult> { - require_admin(authed.is_admin, &authed.username)?; let mut tx = user_db.begin(&authed).await?; let jobs_to_cancel = sqlx::query_scalar!( diff --git a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte index 864ecd8fe9..fc6326f0ec 100644 --- a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte @@ -473,6 +473,7 @@ selectedIds = [] jobLoader?.loadJobs(minTs, maxTs, true, true) sendUserToast(`Canceled ${uuids.length} jobs`) + isSelectingJobsToCancel = false }} loading={fetchingFilteredJobs} on:canceled={() => { @@ -496,6 +497,7 @@ selectedIds = [] jobLoader?.loadJobs(minTs, maxTs, true, true) sendUserToast(`Canceled ${uuids.length} jobs`) + isSelectingJobsToCancel = false }} on:canceled={() => { isCancelingVisibleJobs = false From 6749f2c1367bdbecb12941baaa87b63c4b4c6f20 Mon Sep 17 00:00:00 2001 From: Faton Ramadani Date: Tue, 6 Aug 2024 10:59:49 +0200 Subject: [PATCH 08/13] fix(frontend): Remove full height for the event handlers of runnables (#4196) --- .../src/lib/components/apps/editor/SettingsPanel.svelte | 1 + .../apps/editor/settingsPanel/common/PanelSection.svelte | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/apps/editor/SettingsPanel.svelte b/frontend/src/lib/components/apps/editor/SettingsPanel.svelte index df16b59593..2f0792308f 100644 --- a/frontend/src/lib/components/apps/editor/SettingsPanel.svelte +++ b/frontend/src/lib/components/apps/editor/SettingsPanel.svelte @@ -156,6 +156,7 @@ {/if} From b9b30e66ec485cc019561b092fc27d08aa4666b3 Mon Sep 17 00:00:00 2001 From: Faton Ramadani Date: Tue, 6 Aug 2024 12:20:16 +0200 Subject: [PATCH 09/13] fix(frontend): Hide AgChart background to make styling work (#4197) * fix(frontend): Hide AgChart background to make styling work * fix(frontend): Fix dark theme --- .../display/charts/AppAgCharts.svelte | 63 ++++++++++++++++++- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/apps/components/display/charts/AppAgCharts.svelte b/frontend/src/lib/components/apps/components/display/charts/AppAgCharts.svelte index 4b8fe6fa0e..83d351fbb2 100644 --- a/frontend/src/lib/components/apps/components/display/charts/AppAgCharts.svelte +++ b/frontend/src/lib/components/apps/components/display/charts/AppAgCharts.svelte @@ -8,13 +8,14 @@ RichConfigurations } from '../../../types' import { initCss } from '../../../utils' - import { getContext } from 'svelte' + import { getContext, tick } from 'svelte' import { initConfig, initOutput } from '../../../editor/appUtils' import { components } from '../../../editor/component' import ResolveConfig from '../../helpers/ResolveConfig.svelte' import { twMerge } from 'tailwind-merge' import ResolveStyle from '../../helpers/ResolveStyle.svelte' import type { AgChartOptions, AgChartInstance } from 'ag-charts-community' + import DarkModeObserver from '$lib/components/DarkModeObserver.svelte' export let id: string export let componentInput: AppInput | undefined @@ -58,6 +59,48 @@ let css = initCss($app.css?.agchartscomponent, customCss) let chartInstance: AgChartInstance | undefined = undefined + function getChartStyleByTheme() { + const gridColor = darkMode ? '#555555' : '#dddddd' + const axisColor = darkMode ? '#555555' : '#dddddd' + const textColor = darkMode ? '#eeeeee' : '#333333' + + return { + axes: [ + { + type: 'category', + position: 'bottom', + label: { color: textColor }, + line: { color: axisColor }, + tick: { color: axisColor }, + gridLine: { + style: [ + { + stroke: gridColor + } + ] + } + }, + { + type: 'number', + position: 'left', + label: { color: textColor }, + line: { color: axisColor }, + tick: { color: axisColor }, + gridLine: { + style: [ + { + stroke: gridColor + } + ] + } + } + ], + background: { + visible: false + } + } + } + function updateChart() { if (!chartInstance) { return @@ -111,7 +154,8 @@ yName: d.name } } - }) as any[]) ?? [] + }) as any[]) ?? [], + ...getChartStyleByTheme() } outputs.result.set({ @@ -220,6 +264,7 @@ } const options = { container: document.getElementById(`agchart-${id}`) as HTMLElement, + ...getChartStyleByTheme(), ...result } @@ -252,7 +297,8 @@ const options: AgChartOptions = { container: document.getElementById(`agchart-${id}`) as HTMLElement, data: [], - series: [] + series: [], + ...getChartStyleByTheme() } chartInstance = AgChartsInstance?.create(options) @@ -274,8 +320,19 @@ initChart() }) } + + let darkMode = false + { + tick().then(() => { + updateChart() + }) + }} +/> + {#if datasets} Date: Tue, 6 Aug 2024 09:07:24 -0400 Subject: [PATCH 10/13] bump go version (#4192) * bump go version * update go version in lsp --- Dockerfile | 8 ++++---- lsp/Dockerfile | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 30252632e0..1044244797 100644 --- a/Dockerfile +++ b/Dockerfile @@ -127,13 +127,13 @@ RUN set -eux; \ arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \ case "$arch" in \ 'amd64') \ - targz='go1.21.6.linux-amd64.tar.gz'; \ + targz='go1.22.5.linux-amd64.tar.gz'; \ ;; \ 'arm64') \ - targz='go1.21.6.linux-arm64.tar.gz'; \ + targz='go1.22.5.linux-arm64.tar.gz'; \ ;; \ 'armhf') \ - targz='go1.21.6.linux-armv6l.tar.gz'; \ + targz='go1.22.5.linux-armv6l.tar.gz'; \ ;; \ *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \ esac; \ @@ -173,4 +173,4 @@ RUN windmill cache EXPOSE 8000 -CMD ["windmill"] \ No newline at end of file +CMD ["windmill"] diff --git a/lsp/Dockerfile b/lsp/Dockerfile index 88cc424f67..34aec5750d 100644 --- a/lsp/Dockerfile +++ b/lsp/Dockerfile @@ -9,13 +9,13 @@ RUN set -eux; \ url=; \ case "$arch" in \ 'amd64') \ - targz='go1.21.0.linux-amd64.tar.gz'; \ + targz='go1.22.5.linux-amd64.tar.gz'; \ ;; \ 'arm64') \ - targz='go1.21.0.linux-arm64.tar.gz'; \ + targz='go1.22.5.linux-arm64.tar.gz'; \ ;; \ 'armhf') \ - targz='go1.21.0.linux-armv6l.tar.gz'; \ + targz='go1.22.5.linux-armv6l.tar.gz'; \ ;; \ *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \ esac; \ From 1e7de238cf4c234bf8425ab2c7e57214ea34e8f4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 6 Aug 2024 15:34:57 +0200 Subject: [PATCH 11/13] improve rawvalue handling for lightweightarg input --- .../lib/components/LightweightArgInput.svelte | 15 +++++-------- .../lib/components/StringTypeNarrowing.svelte | 22 +++++++++++++------ frontend/src/lib/utils.ts | 17 +++++++------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/frontend/src/lib/components/LightweightArgInput.svelte b/frontend/src/lib/components/LightweightArgInput.svelte index 0dc9e2d892..5e709893ff 100644 --- a/frontend/src/lib/components/LightweightArgInput.svelte +++ b/frontend/src/lib/components/LightweightArgInput.svelte @@ -96,18 +96,15 @@ $: render && changeDefaultValue(inputCat, defaultValue) - $: rawValue && evalRawValueToValue() + $: (rawValue || inputCat === 'object') && evalRawValueToValue() $: validateInput(pattern, value, required) - $: { - if (inputCat === 'object') { - evalValueToRaw() - } - } - function evalRawValueToValue() { - if (rawValue) { + if (!rawValue || rawValue === '') { + value = undefined + error = '' + } else { try { value = JSON.parse(rawValue) error = '' @@ -124,7 +121,7 @@ } else { // If value is undefined, set rawValue to empty object // This is to prevent the textarea from being empty - rawValue = '{}' + rawValue = '' } } diff --git a/frontend/src/lib/components/StringTypeNarrowing.svelte b/frontend/src/lib/components/StringTypeNarrowing.svelte index 0a2d2a118f..8ca87357e2 100644 --- a/frontend/src/lib/components/StringTypeNarrowing.svelte +++ b/frontend/src/lib/components/StringTypeNarrowing.svelte @@ -27,12 +27,8 @@ export let overrideAllowKindChange: boolean = true export let originalType: string | undefined = undefined - let kind: 'none' | 'pattern' | 'enum' | 'resource' | 'format' | 'base64' = computeKind( - enum_, - contentEncoding, - pattern, - format - ) + let kind: 'none' | 'pattern' | 'enum' | 'resource' | 'format' | 'base64' | 'date-time' = + computeKind(enum_, contentEncoding, pattern, format) const allowKindChange = overrideAllowKindChange || originalType === 'string' @@ -55,6 +51,15 @@ // 'jsonpointer', ] + const FIELD_SETTINGS = [ + ['None', 'none'], + ['File', 'base64', 'Encoded as Base 64'], + ['Enum', 'enum'], + ['Datetime', 'date-time'], + ['Format', 'format'], + ['Pattern', 'pattern'] + ] + $: format = kind == 'resource' ? (resource != undefined ? `resource-${resource}` : 'resource') : format $: pattern = patternStr == '' ? undefined : patternStr @@ -111,6 +116,9 @@ if (e.detail != 'enum') { enum_ = undefined } + if (e.detail == 'date-time') { + format = 'date-time' + } if (e.detail == 'none') { pattern = undefined format = undefined @@ -122,7 +130,7 @@ } }} > - {#each [['None', 'none'], ['File', 'base64', 'Encoded as Base 64'], ['Enum', 'enum'], ['Format', 'format'], ['Pattern', 'pattern']] as x} + {#each FIELD_SETTINGS as x} {/each} diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 876d60bce2..896e3a73a4 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -495,7 +495,7 @@ export function isObject(obj: any) { export function debounce(func: (...args: any[]) => any, wait: number) { let timeout: any - return function(...args: any[]) { + return function (...args: any[]) { // @ts-ignore const context = this clearTimeout(timeout) @@ -505,7 +505,7 @@ export function debounce(func: (...args: any[]) => any, wait: number) { export function throttle(func: (...args: any[]) => T, wait: number) { let timeout: any - return function(...args: any[]) { + return function (...args: any[]) { if (!timeout) { timeout = setTimeout(() => { timeout = null @@ -721,7 +721,7 @@ export async function tryEvery({ try { await tryCode() break - } catch (err) { } + } catch (err) {} i++ } if (i >= times) { @@ -883,7 +883,7 @@ export function computeKind( contentEncoding: 'base64' | 'binary' | undefined, pattern: string | undefined, format: string | undefined -): 'base64' | 'none' | 'pattern' | 'enum' | 'resource' | 'format' { +): 'base64' | 'none' | 'pattern' | 'enum' | 'resource' | 'format' | 'date-time' { if (enum_ != undefined) { return 'enum' } @@ -893,6 +893,9 @@ export function computeKind( if (pattern != undefined) { return 'pattern' } + if (format == 'date-time') { + return 'date-time' + } if (format != undefined && format != '') { if (format?.startsWith('resource')) { return 'resource' @@ -950,10 +953,7 @@ export function isDeployable( return false } - if ( - deployUiSettings.include_type != undefined && - !deployUiSettings.include_type.includes(type) - ) { + if (deployUiSettings.include_type != undefined && !deployUiSettings.include_type.includes(type)) { return false } @@ -972,4 +972,3 @@ export const ALL_DEPLOYABLE: WorkspaceDeployUISettings = { include_path: [], include_type: ['script', 'flow', 'app', 'resource', 'variable', 'secret'] } - From 7886f8f471bcb33f7cc640eab64ffc0dd3cb1726 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 6 Aug 2024 16:14:52 +0200 Subject: [PATCH 12/13] fix: fix native scripts access to reserved variables --- backend/windmill-worker/src/bun_executor.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 4e80f30067..9f665e1009 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -967,11 +967,10 @@ try {{ }; let reserved_variables_args_out_f = async { - if annotation.native_mode { - return Ok(HashMap::new()) as error::Result>; - } let args_and_out_f = async { - create_args_and_out_file(&client, job, job_dir, db).await?; + if !annotation.native_mode { + create_args_and_out_file(&client, job, job_dir, db).await?; + } Ok(()) as Result<()> }; let reserved_variables_f = async { From a87f34fb4a5e54da38e2dbabf2ac551d35b021ee Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Tue, 6 Aug 2024 16:17:33 +0200 Subject: [PATCH 13/13] feat: email triggers (#4163) * feat: email triggers v0 * update docker compose to nginx with tcp reverse proxy + move smtp to private * fix: open source build * test: update ee ref for testing * feat: use caddy with layer4 * fix: nit * feat: configurable email domain * fix: nit * fix: nit * fix: get l4 from main * fix: default email domain to mail.domain * update ee ref --- Caddyfile | 10 +++ backend/Cargo.lock | 14 ++++ backend/Cargo.toml | 3 + backend/ee-repo-ref.txt | 2 +- backend/src/main.rs | 1 + backend/tests/worker.rs | 1 + backend/windmill-api/Cargo.toml | 4 + backend/windmill-api/src/jobs.rs | 81 ++++++++++++++++++- backend/windmill-api/src/lib.rs | 20 ++++- backend/windmill-api/src/settings.rs | 6 +- backend/windmill-api/src/smtp_server_ee.rs | 17 ++++ .../windmill-common/src/global_settings.rs | 1 + backend/windmill-worker/src/worker.rs | 28 +++++++ docker-compose.yml | 5 +- .../components/details/WebhooksPanel.svelte | 65 +++++++++++++-- .../src/lib/components/instanceSettings.ts | 9 +++ 16 files changed, 248 insertions(+), 19 deletions(-) create mode 100644 backend/windmill-api/src/smtp_server_ee.rs diff --git a/Caddyfile b/Caddyfile index 8d3cc96265..67925e6492 100644 --- a/Caddyfile +++ b/Caddyfile @@ -1,3 +1,13 @@ +{ + layer4 { + :25 { + proxy { + to windmill_server:2525 + } + } + } +} + {$BASE_URL} { bind {$ADDRESS} reverse_proxy /ws/* http://lsp:3001 diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 4d34ff0e8c..d9bf8090b5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -4856,6 +4856,16 @@ dependencies = [ "gethostname", ] +[[package]] +name = "mail-parser" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5a1335c3a964788c90cb42ae04a34b5f2628e89566949ce3bd4ada695c0bcd" +dependencies = [ + "encoding_rs", + "serde", +] + [[package]] name = "mail-send" version = "0.4.9" @@ -10455,9 +10465,12 @@ dependencies = [ "jsonwebtoken", "lazy_static", "magic-crypt", + "mail-parser", "mime_guess", + "native-tls", "object_store", "openidconnect", + "openssl", "pin-project", "prometheus", "quick_cache", @@ -10479,6 +10492,7 @@ dependencies = [ "tinyvector", "tokenizers", "tokio", + "tokio-native-tls", "tokio-tar", "tokio-util", "tower", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index aebecc37b7..e8f53be76f 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -240,6 +240,9 @@ candle-nn = "0.3.0" tiberius = { git = "https://github.com/prisma/tiberius", rev = "8f66a699dfa041e7b5f736c7e94f92c945453c9e", default-features = false, features = ["rustls", "tds73", "chrono", "sql-browser-tokio"]} pin-project = "1" indexmap = { version = "2.2.5", features = ["serde"]} +tokio-native-tls = "^0" +openssl = "=0.10" +mail-parser = "^0" datafusion = "39.0.0" object_store = { version = "0.10.0", features = ["aws", "azure"] } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 428f05b845..e02e4aa6a6 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -1a2febc371c907789b25860366872fca888d6477 +70e475a5cec356d2fe992038f303aea8988b5046 \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index 90758997cb..c26695d7c5 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -405,6 +405,7 @@ Windmill Community Edition {GIT_VERSION} server_killpill_rx, base_internal_tx, server_mode, + base_internal_url.clone(), ) .await?; } else { diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 48eb9f2642..7ed4e56085 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -128,6 +128,7 @@ impl ApiServer { rx, port_tx, false, + format!("http://localhost:{}", addr.port()), )); _port_rx.await.unwrap(); diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index a443a19698..ce5addbd38 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -59,6 +59,10 @@ tracing-subscriber.workspace = true quick_cache.workspace = true rand.workspace = true time.workspace = true +native-tls.workspace = true +tokio-native-tls.workspace = true +openssl.workspace = true +mail-parser = { workspace = true, features = ["serde_support"] } magic-crypt.workspace = true tempfile.workspace = true tokio-util.workspace = true diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 706f588c9c..c102ec6fb4 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -1034,7 +1034,7 @@ pub struct ListableCompletedJob { pub labels: Option, } -#[derive(Deserialize, Clone)] +#[derive(Deserialize, Clone, Default)] pub struct RunJobQuery { scheduled_for: Option>, scheduled_in_secs: Option, @@ -2751,6 +2751,23 @@ pub async fn run_flow_by_path( Path((w_id, flow_path)): Path<(String, StripPath)>, Query(run_query): Query, args: PushArgsOwned, +) -> error::Result<(StatusCode, String)> { + run_flow_by_path_inner( + authed, db, user_db, rsmq, w_id, flow_path, run_query, args, None, + ) + .await +} + +pub async fn run_flow_by_path_inner( + authed: ApiAuthed, + db: DB, + user_db: UserDB, + rsmq: Option, + w_id: String, + flow_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + label_prefix: Option, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -2784,7 +2801,9 @@ pub async fn run_flow_by_path( &w_id, JobPayload::Flow { path: flow_path.to_string(), dedicated_worker }, PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), + &label_prefix + .map(|x| x + authed.display_username()) + .unwrap_or_else(|| authed.display_username().to_string()), &authed.email, username_to_permissioned_as(&authed.username), scheduled_for, @@ -2910,6 +2929,31 @@ pub async fn run_script_by_path( Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, args: PushArgsOwned, +) -> error::Result<(StatusCode, String)> { + run_script_by_path_inner( + authed, + db, + user_db, + rsmq, + w_id, + script_path, + run_query, + args, + None, + ) + .await +} + +pub async fn run_script_by_path_inner( + authed: ApiAuthed, + db: DB, + user_db: UserDB, + rsmq: Option, + w_id: String, + script_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + label_prefix: Option, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -2935,7 +2979,9 @@ pub async fn run_script_by_path( &w_id, job_payload, PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), + &label_prefix + .map(|x| x + authed.display_username()) + .unwrap_or_else(|| authed.display_username().to_string()), &authed.email, username_to_permissioned_as(&authed.username), scheduled_for, @@ -4352,6 +4398,31 @@ pub async fn run_job_by_hash( Path((w_id, script_hash)): Path<(String, ScriptHash)>, Query(run_query): Query, args: PushArgsOwned, +) -> error::Result<(StatusCode, String)> { + run_job_by_hash_inner( + authed, + db, + user_db, + rsmq, + w_id, + script_hash, + run_query, + args, + None, + ) + .await +} + +pub async fn run_job_by_hash_inner( + authed: ApiAuthed, + db: DB, + user_db: UserDB, + rsmq: Option, + w_id: String, + script_hash: ScriptHash, + run_query: RunJobQuery, + args: PushArgsOwned, + label_prefix: Option, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -4398,7 +4469,9 @@ pub async fn run_job_by_hash( priority, }, PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), + &label_prefix + .map(|x| x + authed.display_username()) + .unwrap_or_else(|| authed.display_username().to_string()), &authed.email, username_to_permissioned_as(&authed.username), scheduled_for, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 1540395ed4..ed842aeb42 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -12,6 +12,7 @@ use crate::ee::ExternalJwks; #[cfg(feature = "embedding")] use crate::embeddings::load_embeddings_db; use crate::oauth2_ee::AllClients; +use crate::smtp_server_ee::SmtpServer; use crate::tracing_init::MyOnFailure; use crate::{ oauth2_ee::SlackVerifier, @@ -76,6 +77,7 @@ mod schedule; mod scim_ee; mod scripts; mod settings; +pub mod smtp_server_ee; mod static_assets; mod stripe_ee; mod tracing_init; @@ -163,6 +165,7 @@ pub async fn run_server( mut rx: tokio::sync::broadcast::Receiver<()>, port_tx: tokio::sync::oneshot::Sender, server_mode: bool, + base_internal_url: String, ) -> anyhow::Result<()> { if let Some(mut rsmq) = rsmq.clone() { for tag in ALL_TAGS.read().await.iter() { @@ -194,8 +197,8 @@ pub async fn run_server( let middleware_stack = ServiceBuilder::new() .layer(Extension(db.clone())) - .layer(Extension(rsmq)) - .layer(Extension(user_db)) + .layer(Extension(rsmq.clone())) + .layer(Extension(user_db.clone())) .layer(Extension(auth_cache.clone())) .layer(Extension(index_reader)) .layer(Extension(index_writer)) @@ -214,7 +217,18 @@ pub async fn run_server( if server_mode { #[cfg(feature = "embedding")] - load_embeddings_db(&db) + load_embeddings_db(&db); + + let smtp_server = Arc::new(SmtpServer { + db: db.clone(), + user_db: user_db, + auth_cache: auth_cache.clone(), + rsmq: rsmq, + base_internal_url: base_internal_url.clone(), + }); + if let Err(err) = smtp_server.start_listener_thread(addr).await { + tracing::error!("Error starting SMTP server: {err:#}"); + } } let job_helpers_service = { diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 6650b60ae5..c0a8c2960f 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -24,7 +24,10 @@ use axum::{ use serde::Deserialize; use windmill_common::{ error::{self, JsonResult, Result}, - global_settings::{AUTOMATE_USERNAME_CREATION_SETTING, ENV_SETTINGS, HUB_BASE_URL_SETTING}, + global_settings::{ + AUTOMATE_USERNAME_CREATION_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, + HUB_BASE_URL_SETTING, + }, server::Smtp, utils::send_email, }; @@ -256,6 +259,7 @@ pub async fn get_global_setting( && !key.starts_with("default_recovery_handler_") && key != AUTOMATE_USERNAME_CREATION_SETTING && key != HUB_BASE_URL_SETTING + && key != EMAIL_DOMAIN_SETTING { require_super_admin(&db, &authed.email).await?; } diff --git a/backend/windmill-api/src/smtp_server_ee.rs b/backend/windmill-api/src/smtp_server_ee.rs new file mode 100644 index 0000000000..88837c5974 --- /dev/null +++ b/backend/windmill-api/src/smtp_server_ee.rs @@ -0,0 +1,17 @@ +use crate::{db::DB, users::AuthCache}; +use std::{net::SocketAddr, sync::Arc}; +use windmill_common::db::UserDB; + +pub struct SmtpServer { + pub auth_cache: Arc, + pub db: DB, + pub user_db: UserDB, + pub rsmq: Option, + pub base_internal_url: String, +} + +impl SmtpServer { + pub async fn start_listener_thread(self: Arc, _addr: SocketAddr) -> anyhow::Result<()> { + Err(anyhow::anyhow!("Implementation not open source")) + } +} diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 6e86646dcc..a49b755ddb 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -28,6 +28,7 @@ pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; +pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const ENV_SETTINGS: [&str; 50] = [ "DISABLE_NSJAIL", diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index e274f1790d..af4b4a9234 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -2278,6 +2278,34 @@ async fn handle_queued_job( return Err(Error::ExecutionErr(e.to_string())); } + #[cfg(not(feature = "enterprise"))] + if job.created_by.starts_with("email-trigger-") { + let daily_count = sqlx::query!( + "SELECT value FROM metrics WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day' ORDER BY created_at DESC LIMIT 1" + ).fetch_optional(db).await?.map(|x| serde_json::from_value::(x.value).unwrap_or(1)); + + if let Some(count) = daily_count { + if count >= 100 { + return Err(error::Error::QuotaExceeded(format!( + "Email trigger usage limit of 100 per day has been reached." + ))); + } else { + sqlx::query!( + "UPDATE metrics SET value = $1 WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day'", + serde_json::json!(count + 1) + ) + .execute(db) + .await?; + } + } else { + sqlx::query!( + "INSERT INTO metrics (id, value) VALUES ('email_trigger_usage', to_jsonb(1))" + ) + .execute(db) + .await?; + } + } + let step = if job.is_flow_step { let r = update_flow_status_in_progress( db, diff --git a/docker-compose.yml b/docker-compose.yml index a867f9a292..0aefa45496 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ services: restart: unless-stopped expose: - 8000 + - 2525 environment: - DATABASE_URL=${DATABASE_URL} - MODE=server @@ -154,9 +155,8 @@ services: - 3002 caddy: - image: caddy:2.5.2-alpine + image: ghcr.io/windmill-labs/caddy-l4:latest restart: unless-stopped - # Configure the mounted Caddyfile and the exposed ports or use another reverse proxy if needed volumes: - ./Caddyfile:/etc/caddy/Caddyfile @@ -164,6 +164,7 @@ services: ports: # To change the exposed port, simply change 80:80 to :80. No other changes needed - 80:80 + - 25:25 # - 443:443 # Uncomment to enable HTTPS handling by Caddy environment: - BASE_URL=":80" diff --git a/frontend/src/lib/components/details/WebhooksPanel.svelte b/frontend/src/lib/components/details/WebhooksPanel.svelte index 24dac2fba5..963f70f055 100644 --- a/frontend/src/lib/components/details/WebhooksPanel.svelte +++ b/frontend/src/lib/components/details/WebhooksPanel.svelte @@ -18,6 +18,8 @@ import ClipboardPanel from './ClipboardPanel.svelte' import { copyToClipboard, generateRandomString } from '$lib/utils' import HighlightTheme from '../HighlightTheme.svelte' + import Alert from '../common/alert/Alert.svelte' + import { SettingService } from '$lib/gen' let userSettings: UserSettings @@ -28,6 +30,8 @@ export let hash: string | undefined = undefined export let path: string + let selectedTab: string = 'rest' + let webhooks: { async: { hash?: string @@ -40,6 +44,15 @@ } } + let emailDomain: string = "mail." + $page.url.hostname + async function getEmailDomain() { + emailDomain = + ((await SettingService.getGlobal({ + key: 'email_domain' + })) as any) ?? ("mail." + $page.url.hostname) + } + getEmailDomain() + $: webhooks = isFlow ? computeFlowWebhooks(path) : computeScriptWebhooks(hash, path) function computeScriptWebhooks(hash: string | undefined, path: string) { @@ -82,6 +95,10 @@ requestType = 'hash' } + $: if (webhookType === 'sync' && selectedTab === 'email') { + webhookType = 'async' + } + $: url = webhooks[webhookType][requestType] + (tokenType === 'query' @@ -108,6 +125,12 @@ return headers } + function emailAddress() { + return `${$workspaceStore}+${ + requestType === 'hash' ? 'hash.' + hash : (isFlow ? 'flow.' : '') + path.replaceAll('/', '.') + }+${token}@${emailDomain}` + } + function fetchCode() { if (webhookType === 'sync') { return ` @@ -261,6 +284,7 @@ done` label="Sync" value="sync" tooltip="Triggers the execution, wait for the job to complete and return it as a response." + disabled={selectedTab === 'email'} />
@@ -291,22 +315,25 @@ done` />
-
-
Token configuration
- - - - -
+ {#if selectedTab !== 'email'} +
+
Token configuration
+ + + + +
+ {/if} - + REST {#if SCRIPT_VIEW_SHOW_EXAMPLE_CURL} Curl {/if} Fetch + Email {#key token} @@ -365,6 +392,28 @@ done` {/key}{/key}{/key}{/key} {/key} + +
+ {#key args} + {#key requestType} + {#key webhookType} + {#key tokenType} + {#key token} +
+ +
+ {/key} + {/key} + {/key} + {/key} + {/key} + + To trigger the job by email, send an email to the address above. The job will receive + two arguments: `raw_email` containing the raw email as string, and `parsed_email` + containing the parsed email as an object. + +
+ {/key} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index da244110c6..5f1d4f765f 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -44,6 +44,15 @@ export const settings: Record = { !value?.endsWith(' ') : false }, + { + label: 'Email domain', + description: + 'Domain to display in webhooks for email triggers, default is the webpage domain prefixed by "mail."', + key: 'email_domain', + fieldType: 'text', + storage: 'setting', + placeholder: 'mail.windmill.com' + }, { label: 'Request Size Limit In MB', description: 'Maximum size of HTTP requests in MB.',