From 1ec6c6f765904361e641d89495890bc87e8544aa Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Wed, 4 Dec 2024 16:50:17 +0100 Subject: [PATCH 01/48] feat: app custom paths (#4828) * feat: app custom paths * nit * make ee only + fix sqlx * fix: custom http routes auth * nits * fix auth + nits * apps_ee * move custom path to ee * fix app jwt * update ee ref --- ...71ca897dcfa9e82618ce8f11afb08a39e3b20.json | 23 +++ ...979f51b53dcd4d8ec50d312c9e7fe31ad5f5.json} | 7 +- ...a1d606d9725e20e9c2d76a0887fadfd87f8df.json | 25 +++ backend/ee-repo-ref.txt | 2 +- .../20241202134622_app_custom_path.down.sql | 2 + .../20241202134622_app_custom_path.up.sql | 2 + backend/windmill-api/openapi.yaml | 50 +++++ backend/windmill-api/src/apps.rs | 109 +++++++++-- backend/windmill-api/src/apps_ee.rs | 5 + backend/windmill-api/src/lib.rs | 13 ++ .../components/apps/editor/AppEditor.svelte | 1 + .../apps/editor/AppEditorHeader.svelte | 126 +++++++++--- .../components/details/ClipboardPanel.svelte | 6 +- .../(logged)/apps/edit/[...path]/+page.svelte | 4 +- frontend/src/routes/a/[...path]/+page.js | 5 + frontend/src/routes/a/[...path]/+page.svelte | 182 ++++++++++++++++++ 16 files changed, 514 insertions(+), 48 deletions(-) create mode 100644 backend/.sqlx/query-1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20.json rename backend/.sqlx/{query-75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2.json => query-6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5.json} (63%) create mode 100644 backend/.sqlx/query-fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df.json create mode 100644 backend/migrations/20241202134622_app_custom_path.down.sql create mode 100644 backend/migrations/20241202134622_app_custom_path.up.sql create mode 100644 backend/windmill-api/src/apps_ee.rs create mode 100644 frontend/src/routes/a/[...path]/+page.js create mode 100644 frontend/src/routes/a/[...path]/+page.svelte diff --git a/backend/.sqlx/query-1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20.json b/backend/.sqlx/query-1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20.json new file mode 100644 index 0000000000..51343088a6 --- /dev/null +++ b/backend/.sqlx/query-1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20" +} diff --git a/backend/.sqlx/query-75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2.json b/backend/.sqlx/query-6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5.json similarity index 63% rename from backend/.sqlx/query-75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2.json rename to backend/.sqlx/query-6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5.json index efaac63d39..0c82c9dd74 100644 --- a/backend/.sqlx/query-75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2.json +++ b/backend/.sqlx/query-6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO app\n (workspace_id, path, summary, policy, versions, draft_only)\n VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id", + "query": "INSERT INTO app\n (workspace_id, path, summary, policy, versions, draft_only, custom_path)\n VALUES ($1, $2, $3, $4, '{}', $5, $6) RETURNING id", "describe": { "columns": [ { @@ -15,12 +15,13 @@ "Varchar", "Varchar", "Jsonb", - "Bool" + "Bool", + "Text" ] }, "nullable": [ false ] }, - "hash": "75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2" + "hash": "6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5" } diff --git a/backend/.sqlx/query-fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df.json b/backend/.sqlx/query-fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df.json new file mode 100644 index 0000000000..a2362be620 --- /dev/null +++ b/backend/.sqlx/query-fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 496432b7e0..c7e750d0f4 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -aefbc1e2188fea312996fcfc30a29d8fb5315316 \ No newline at end of file +8606d98a692d11b09a387c5efbd6b4335c533fd3 \ No newline at end of file diff --git a/backend/migrations/20241202134622_app_custom_path.down.sql b/backend/migrations/20241202134622_app_custom_path.down.sql new file mode 100644 index 0000000000..1dadbefff1 --- /dev/null +++ b/backend/migrations/20241202134622_app_custom_path.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE app DROP COLUMN custom_path; diff --git a/backend/migrations/20241202134622_app_custom_path.up.sql b/backend/migrations/20241202134622_app_custom_path.up.sql new file mode 100644 index 0000000000..832efd0cb2 --- /dev/null +++ b/backend/migrations/20241202134622_app_custom_path.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE app ADD COLUMN custom_path TEXT CHECK (custom_path ~ '^[\w-]+(\/[\w-]+)*$'); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4a9492418b..f0bd8856d5 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -3733,6 +3733,27 @@ paths: required: - app + /apps_u/public_app_by_custom_path/{custom_path}: + get: + summary: get public app by custom path + operationId: getPublicAppByCustomPath + tags: + - app + parameters: + - $ref: "#/components/parameters/CustomPath" + responses: + "200": + description: app details + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/AppWithLastVersion" + - type: object + properties: + workspace_id: + type: string + /scripts/hub/get/{path}: get: summary: get hub script content by path @@ -5371,6 +5392,8 @@ paths: type: boolean deployment_message: type: string + custom_path: + type: string required: - path - value @@ -5696,6 +5719,8 @@ paths: $ref: "#/components/schemas/Policy" deployment_message: type: string + custom_path: + type: string responses: "200": description: app updated @@ -5704,6 +5729,23 @@ paths: schema: type: string + /w/{workspace}/apps/custom_path_exists/{custom_path}: + get: + summary: check if custom path exists + operationId: customPathExists + tags: + - app + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/CustomPath" + responses: + "200": + description: custom path exists + content: + application/json: + schema: + type: boolean + /w/{workspace}/apps_u/execute_component/{path}: post: summary: executeComponent @@ -10222,6 +10264,12 @@ components: required: true schema: type: string + CustomPath: + name: custom_path + in: path + required: true + schema: + type: string PathId: name: id in: path @@ -12860,6 +12908,8 @@ components: draft_only: type: boolean draft: {} + custom_path: + type: string AppHistory: type: object diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 5e236baeb6..e1a9421252 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -56,10 +56,10 @@ use windmill_common::{ jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode}, users::username_to_permissioned_as, utils::{ - http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath, + http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath }, variables::{build_crypt, build_crypt_with_key_suffix}, - worker::to_raw_value, + worker::{to_raw_value, CLOUD_HOSTED}, HUB_BASE_URL, }; @@ -81,6 +81,7 @@ pub fn workspaced_service() -> Router { .route("/history/p/*path", get(get_app_history)) .route("/get_latest_version/*path", get(get_latest_version)) .route("/history_update/a/:id/v/:version", post(update_app_history)) + .route("/custom_path_exists/*custom_path", get(custom_path_exists)) } pub fn unauthed_service() -> Router { @@ -90,13 +91,17 @@ pub fn unauthed_service() -> Router { .route("/public_app/:secret", get(get_public_app_by_secret)) .route("/public_resource/*path", get(get_public_resource)) } - pub fn global_service() -> Router { Router::new() .route("/hub/list", get(list_hub_apps)) .route("/hub/get/:id", get(get_hub_app_by_id)) } +#[cfg(not(feature = "enterprise"))] +pub fn global_unauthed_service() -> Router { + Router::new() +} + #[derive(FromRow, Deserialize, Serialize)] pub struct ListableApp { pub id: i64, @@ -147,21 +152,26 @@ pub struct AppWithLastVersionAndStarred { pub starred: Option, } +#[cfg(feature = "enterprise")] +#[derive(Serialize, FromRow)] +pub struct AppWithLastVersionAndWorkspace { + #[sqlx(flatten)] + #[serde(flatten)] + pub app: AppWithLastVersion, + pub workspace_id: String, +} + #[derive(Serialize, Deserialize, FromRow)] pub struct AppWithLastVersionAndDraft { - pub id: i64, - pub path: String, - pub summary: String, - pub policy: sqlx::types::Json>, - pub versions: Vec, - pub value: sqlx::types::Json>, - pub created_by: String, - pub created_at: chrono::DateTime, - pub extra_perms: serde_json::Value, + #[sqlx(flatten)] + #[serde(flatten)] + pub app: AppWithLastVersion, #[serde(skip_serializing_if = "Option::is_none")] pub draft: Option>>, #[serde(skip_serializing_if = "Option::is_none")] pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_path: Option, } #[derive(Serialize)] @@ -229,6 +239,7 @@ pub struct CreateApp { pub policy: Policy, pub draft_only: Option, pub deployment_message: Option, + pub custom_path: Option, } #[derive(Deserialize)] @@ -238,6 +249,7 @@ pub struct EditApp { pub value: Option>>, pub policy: Option, pub deployment_message: Option, + pub custom_path: Option, } #[derive(Serialize, FromRow)] @@ -408,7 +420,7 @@ async fn get_app_w_draft( let app_o = sqlx::query_as::<_, AppWithLastVersionAndDraft>( r#"SELECT app.id, app.path, app.summary, app.versions, app.policy, - app.extra_perms, app_version.value, + app.extra_perms, app_version.value, app.custom_path, app_version.created_at, app_version.created_by, app.draft_only, draft.value as "draft" from app @@ -515,6 +527,22 @@ async fn update_app_history( return Ok(()); } + +async fn custom_path_exists( + Extension(db): Extension, + Path((w_id, custom_path)): Path<(String, String)>, +) -> JsonResult { + let exists = + sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))", + custom_path, + if *CLOUD_HOSTED { Some(&w_id) } else { None } + ) + .fetch_one(&db) + .await?.unwrap_or(false); + Ok(Json(exists)) +} + async fn get_app_by_id( authed: ApiAuthed, Extension(user_db): Extension, @@ -598,6 +626,7 @@ async fn get_public_app_by_secret( Ok(Json(app)) } + async fn get_public_resource( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, @@ -680,6 +709,26 @@ async fn create_app( ))); } + if let Some(custom_path) = &app.custom_path { + + require_admin(authed.is_admin, &authed.username)?; + + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))", + custom_path, + if *CLOUD_HOSTED { Some(&w_id) } else { None } + ) + .fetch_one(&mut *tx) + .await?.unwrap_or(false); + + if exists { + return Err(Error::BadRequest(format!( + "App with custom path {} already exists", + custom_path + ))); + } + } + sqlx::query!( "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'", &app.path, @@ -690,13 +739,14 @@ async fn create_app( let id = sqlx::query_scalar!( "INSERT INTO app - (workspace_id, path, summary, policy, versions, draft_only) - VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id", + (workspace_id, path, summary, policy, versions, draft_only, custom_path) + VALUES ($1, $2, $3, $4, '{}', $5, $6) RETURNING id", w_id, app.path, app.summary, json!(app.policy), app.draft_only, + app.custom_path, ) .fetch_one(&mut *tx) .await?; @@ -899,7 +949,11 @@ async fn update_app( let mut tx = user_db.clone().begin(&authed).await?; - let npath = if ns.policy.is_some() || ns.path.is_some() || ns.summary.is_some() { + let npath = if ns.policy.is_some() + || ns.path.is_some() + || ns.summary.is_some() + || ns.custom_path.is_some() + { let mut sqlb = SqlBuilder::update_table("app"); sqlb.and_where_eq("path", "?".bind(&path)); sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); @@ -932,6 +986,29 @@ async fn update_app( sqlb.set_str("summary", nsummary); } + if let Some(ncustom_path) = &ns.custom_path { + + require_admin(authed.is_admin, &authed.username)?; + + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))", + ncustom_path, + if *CLOUD_HOSTED { Some(&w_id) } else { None }, + path, + w_id + ) + .fetch_one(&mut *tx) + .await?.unwrap_or(false); + + if exists { + return Err(Error::BadRequest(format!( + "App with custom path {} already exists", + ncustom_path + ))); + } + sqlb.set_str("custom_path", ncustom_path); + } + if let Some(mut npolicy) = ns.policy { npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); npolicy.on_behalf_of_email = Some(authed.email.clone()); diff --git a/backend/windmill-api/src/apps_ee.rs b/backend/windmill-api/src/apps_ee.rs new file mode 100644 index 0000000000..a7737664b9 --- /dev/null +++ b/backend/windmill-api/src/apps_ee.rs @@ -0,0 +1,5 @@ +use axum::Router; + +pub fn global_unauthed_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 28f2523ad5..88243789b4 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -64,6 +64,8 @@ mod indexer_ee; mod inputs; mod integration; +#[cfg(feature = "enterprise")] +mod apps_ee; #[cfg(feature = "parquet")] mod job_helpers_ee; pub mod job_metrics; @@ -343,6 +345,17 @@ pub async fn run_server( ) .nest("/concurrency_groups", concurrency_groups::global_service()) .nest("/scripts_u", scripts::global_unauthed_service()) + .nest("/apps_u", { + #[cfg(feature = "enterprise")] + { + apps_ee::global_unauthed_service() + } + + #[cfg(not(feature = "enterprise"))] + { + Router::new() + } + }) .nest( "/w/:workspace_id/apps_u", apps::unauthed_service() diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index 66b2d3e009..f0380f09e2 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -68,6 +68,7 @@ summary: string policy: any draft_only?: boolean + custom_path?: string } | undefined = undefined export let version: number | undefined = undefined diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index 82e47a0473..3c7783e3f4 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -11,12 +11,11 @@ import Toggle from '$lib/components/Toggle.svelte' import { AppService, DraftService, type Job, type Policy } from '$lib/gen' import { redo, undo } from '$lib/history' - import { enterpriseLicense, workspaceStore } from '$lib/stores' + import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' import { AlignHorizontalSpaceAround, BellOff, Bug, - Clipboard, DiffIcon, Expand, FileJson, @@ -39,7 +38,6 @@ import { classNames, cleanValueProperties, - copyToClipboard, truncateRev, orderedJsonStringify, type Value, @@ -90,6 +88,9 @@ import HideButton from './settingsPanel/HideButton.svelte' import DeployOverrideConfirmationModal from '$lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte' import { computeS3FileInputPolicy, computeWorkspaceS3FileInputPolicy } from './appUtilsS3' + import { isCloudHosted } from '$lib/cloud' + import { base } from '$lib/base' + import ClipboardPanel from '$lib/components/details/ClipboardPanel.svelte' async function hash(message) { try { @@ -119,6 +120,7 @@ summary: string policy: any draft_only?: boolean + custom_path?: string } | undefined = undefined export let version: number | undefined = undefined @@ -479,14 +481,16 @@ summary: $summary, policy, path: npath, - deployment_message: deploymentMsg + deployment_message: deploymentMsg, + custom_path: $userStore?.is_admin || $userStore?.is_super_admin ? customPath : undefined } }) savedApp = { summary: $summary, value: structuredClone($app), path: npath, - policy + policy, + custom_path: customPath } const appHistory = await AppService.getAppHistoryByPath({ workspace: $workspaceStore!, @@ -885,6 +889,37 @@ let priorDarkMode = document.documentElement.classList.contains('dark') setTheme($app?.darkMode) + + let customPath = savedApp?.custom_path + let dirtyCustomPath = false + let customPathError = '' + $: fullCustomUrl = `${window.location.origin}${base}/a/${ + isCloudHosted() ? $workspaceStore + '/' : '' + }${customPath}` + async function appExists(customPath: string) { + return await AppService.customPathExists({ + workspace: $workspaceStore!, + customPath + }) + } + let validateTimeout: NodeJS.Timeout | undefined = undefined + async function validateCustomPath(customPath: string): Promise { + customPathError = '' + if (validateTimeout) { + clearTimeout(validateTimeout) + } + validateTimeout = setTimeout(async () => { + if (!/^[\w-]+(\/[\w-]+)*$/.test(customPath)) { + customPathError = 'Invalid path' + } else if (customPath !== savedApp?.custom_path && (await appExists(customPath))) { + customPathError = 'Path already taken' + } else { + customPathError = '' + } + validateTimeout = undefined + }, 500) + } + $: customPath !== undefined && validateCustomPath(customPath) @@ -1071,7 +1106,7 @@ + + {#if loading} + {#if buttonHover} + Stop Refreshing + {:else} + Refreshing... + {/if} + {:else} + Refresh + {/if} + + diff --git a/frontend/src/lib/components/common/layout/List.svelte b/frontend/src/lib/components/common/layout/List.svelte new file mode 100644 index 0000000000..19c5d062cd --- /dev/null +++ b/frontend/src/lib/components/common/layout/List.svelte @@ -0,0 +1,38 @@ + + +{#if horizontal} +
+ +
+{:else} +
+ +
+{/if} diff --git a/frontend/src/lib/components/common/layout/ListElement.svelte b/frontend/src/lib/components/common/layout/ListElement.svelte new file mode 100644 index 0000000000..891079ce42 --- /dev/null +++ b/frontend/src/lib/components/common/layout/ListElement.svelte @@ -0,0 +1,3 @@ +
+ +
diff --git a/frontend/src/lib/components/common/modal/Modal.svelte b/frontend/src/lib/components/common/modal/Modal.svelte index 300499d71a..806f7d33b0 100644 --- a/frontend/src/lib/components/common/modal/Modal.svelte +++ b/frontend/src/lib/components/common/modal/Modal.svelte @@ -70,10 +70,12 @@ >
-

- {title} -

-
+
+

{title}

+ +
+ +
diff --git a/frontend/src/lib/components/common/modal/Modal2.svelte b/frontend/src/lib/components/common/modal/Modal2.svelte new file mode 100644 index 0000000000..7d34172237 --- /dev/null +++ b/frontend/src/lib/components/common/modal/Modal2.svelte @@ -0,0 +1,110 @@ + + + + +{#if isOpen} + +
+
+
{ + close() + }} + > + +
+ +

{title}

+
+ +
+ +
+
+ + +
+ +
+ +
+ +
+ +
+ + + +
{}}> + +
+ +
+
+
+ +{/if} diff --git a/frontend/src/lib/components/common/popup/Popup.svelte b/frontend/src/lib/components/common/popup/Popup.svelte index 5f209b4e50..a559509d15 100644 --- a/frontend/src/lib/components/common/popup/Popup.svelte +++ b/frontend/src/lib/components/common/popup/Popup.svelte @@ -17,6 +17,7 @@ export let target: string | HTMLElement | undefined = undefined export let noTransition = false export let popupHover = false + export let preventPopupClosingOnClickInside = false @@ -28,6 +29,7 @@ +
{ popupHover = false }} + on:click={(e) => preventPopupClosingOnClickInside && e.stopPropagation()} > {#if !noTransition} import { onMount, onDestroy } from 'svelte' import CriticalAlertModalInner from './CriticalAlertModalInner.svelte' - import { SettingService } from '$lib/gen' + import { SettingService, type CriticalAlert } from '$lib/gen' import { sendUserToast } from '$lib/toast' - import { workspaceStore, isCriticalAlertsUIOpen, devopsRole } from '$lib/stores' - import Modal from '../common/modal/Modal.svelte' + import { + workspaceStore, + isCriticalAlertsUIOpen, + devopsRole, + userStore, + superadmin + } from '$lib/stores' + import Modal2 from '../common/modal/Modal2.svelte' + import { Button, Popup } from '$lib/components/common' + import List from '$lib/components/common/layout/List.svelte' + import Toggle from '$lib/components/Toggle.svelte' + import { BellOff, Bell, ExternalLink, Settings } from 'lucide-svelte' + import { base } from '$lib/base' + import Notification from '$lib/components/common/alert/Notification.svelte' export let open: boolean = false export let numUnacknowledgedCriticalAlerts: number = 0 - export let muteSettings + export let muteSettings; let workspaceContext = false + let childRef; $: { setupApiFunctions(workspaceContext) @@ -68,15 +81,50 @@ clearInterval(checkForNewAlertsInterval) }) + async function saveWorkSpaceMuteSetting() { + await SettingService.workspaceMuteCriticalAlertsUi({ + workspace: $workspaceStore!, + requestBody: { + mute_critical_alerts: muteSettings.workspace + } + }) + sendUserToast( + `Critical alert UI mute settings changed.\nPlease reload page for UI changes to take effect.` + ) + childRef.refreshAlerts() + } + async function saveGlobalMuteSetting() { + await SettingService.setGlobal({ + key: 'critical_alert_mute_ui', + requestBody: { value: muteSettings.global } + }) + sendUserToast( + `Critical alert UI mute settings changed.\nPlease reload page for UI changes to take effect.` + ) + childRef.refreshAlerts() + } + async function updateHasUnacknowledgedCriticalAlerts(sendToast: boolean = false) { if (checkingForNewAlerts) return checkingForNewAlerts = true try { - const unacknowledged = await getCriticalAlerts({ + const params = { page: 1, - pageSize: 10, + pageSize: 1000, acknowledged: false - }) + } + let unacknowledged: CriticalAlert[] = [] + if (!$devopsRole && $workspaceStore) { + const res = await SettingService.workspaceGetCriticalAlerts({ + ...params, + workspace: $workspaceStore + }) + unacknowledged = res.alerts ?? [] + } else { + const res = await SettingService.getCriticalAlerts(params) + unacknowledged = res.alerts ?? [] + } + if ( numUnacknowledgedCriticalAlerts === 0 && unacknowledged.length > 0 && @@ -116,14 +164,122 @@ } - + + + + + + + {#if $superadmin || $userStore?.is_admin} + + +
+ +
+ + +
+ {#if $superadmin} + + {/if} +
+ +
+ +
+ + + {/if} + + {#if $superadmin} + + +
+ +
+ + +
+ +
+
+ +
+ + + {:else} + + {/if} + + + - + diff --git a/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte b/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte index c935f82944..5bb4bbdeb3 100644 --- a/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte +++ b/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte @@ -1,14 +1,14 @@ -
-
-
- {#if !hasCriticalAlertChannels && $superadmin} -
- -

- No critical alert channels are set up. Go to the - Instance Settings - page to configure critical alert channels. -

-
- {/if} + + {#if !hasCriticalAlertChannels && $superadmin} +
+ + Go to the + Instance Settings + page to configure critical alert channels. +
+ {/if} -
-
- -
- - {#if $devopsRole} -
- -
- {/if} - - {#if $superadmin || $userStore?.is_admin} -
- {#if $superadmin} -
- -
- {/if} - -
+
+ +
+ + {#if $devopsRole} -
-
- {/if} + {/if} -
-
- -
- + +
-
+ + +
{`${totalNumberOfAlerts === 1000 ? '1000+' : totalNumberOfAlerts ?? '?'} items`} +
+ + +
- -
- - - - - - - {#if $devopsRole} - - {/if} - - - - - {#each alerts as { id, alert_type, message, created_at, acknowledged, workspace_id }} - {#if !hideAcknowledged || !acknowledged} - - - - - - {#if $devopsRole} - - {/if} - - - {/if} - {/each} - -
TypeMessageCreated AtWorkspaceAcknowledge
- {#if alert_type === 'recovered_critical_error'} - - - - {:else} - - - - {/if} - {message}{formatDate(created_at)}{workspace_id ? workspace_id : 'global'} -
- {#if !acknowledged} - - {:else} - - {/if} -
-
-
-
- - Page {page} - -
- - {#if alerts.length === 0} -

No critical alerts available.

- {/if} -
+ + diff --git a/frontend/src/lib/components/sidebar/CriticalAlertTable.svelte b/frontend/src/lib/components/sidebar/CriticalAlertTable.svelte new file mode 100644 index 0000000000..82e9fd3e74 --- /dev/null +++ b/frontend/src/lib/components/sidebar/CriticalAlertTable.svelte @@ -0,0 +1,146 @@ + + +
+ + + +   + Message + Created At + {#if $devopsRole} + Context + {/if} + + + Acked + + + + + + + {#if alerts == undefined} + + {#each new Array(3) as _} + + {#each new Array(5) as _} + + + + {/each} + + {/each} + + {:else if alerts.length === 0} +
+

No critical alerts.

+
+ {:else} + + {#each alerts as { id, alert_type, message, created_at, acknowledged, workspace_id }} + {#if !hideAcknowledged || !acknowledged} + + +
+ {#if alert_type === 'recovered_critical_error'} + + + + {:else} + + + + {/if} +
+ + + +
{message}
+ + + {formatDate(created_at)} + {#if $devopsRole} + {workspace_id ? workspace_id : 'global'} + {/if} + +
+ {#if !acknowledged} + + {:else} + + {/if} +
+ + + {/if} + {/each} + + {/if} + +
diff --git a/frontend/src/lib/components/sidebar/SideBarNotification.svelte b/frontend/src/lib/components/sidebar/SideBarNotification.svelte index d935769972..4e9ea4ba0f 100644 --- a/frontend/src/lib/components/sidebar/SideBarNotification.svelte +++ b/frontend/src/lib/components/sidebar/SideBarNotification.svelte @@ -1,14 +1,12 @@ {#if !small} -
- {notificationCount > 9 ? '9+' : notificationCount} -
+ {:else}
{/if} diff --git a/frontend/src/lib/components/table/Cell.svelte b/frontend/src/lib/components/table/Cell.svelte index 0146ee67c6..ab43a10c2f 100644 --- a/frontend/src/lib/components/table/Cell.svelte +++ b/frontend/src/lib/components/table/Cell.svelte @@ -10,6 +10,7 @@ export let shouldStopPropagation: boolean = false export let selected = false export let sticky: boolean = false + export let wrap: boolean = false let Tag = head ? 'th' : 'td' @@ -24,7 +25,8 @@ if (shouldStopPropagation) e.stopPropagation() }} class={twMerge( - 'text-left text-xs text-primary font-normal whitespace-nowrap', + 'text-left text-xs text-primary font-normal', + wrap ? 'break-words' : 'whitespace-nowrap', first ? 'sm:pl-6' : '', last ? 'sm:pr-6' : '', @@ -33,13 +35,13 @@ numeric ? 'text-right' : '', head ? 'font-semibold ' : '', - $$restProps.class, sticky ? `!p-0 sticky ${first ? 'left-0' : 'right-0'}` : 'px-2 py-3.5', size === 'sm' ? 'px-1.5 py-2.5' : '', size === 'lg' ? 'px-3 py-4' : '', size === 'xs' ? 'px-1 py-1.5' : '', selected ? 'bg-blue-50 dark:bg-blue-900/50' : '', - 'transition-all' + 'transition-all', + $$restProps.class )} > {#if sticky} diff --git a/frontend/src/lib/components/table/DataTable.svelte b/frontend/src/lib/components/table/DataTable.svelte index f9479ab605..e19520cc2e 100644 --- a/frontend/src/lib/components/table/DataTable.svelte +++ b/frontend/src/lib/components/table/DataTable.svelte @@ -9,6 +9,7 @@ import Button from '../common/button/Button.svelte' import { ArrowDownIcon, ArrowLeftIcon, ArrowRightIcon } from 'lucide-svelte' import { twMerge } from 'tailwind-merge' + import List from '$lib/components/common/layout/List.svelte' export let paginated: boolean = false export let currentPage: number = 1 @@ -21,80 +22,87 @@ export let shouldHidePagination: boolean = false export let noBorder: boolean = false export let rowCount: number | undefined = undefined + export let hasMore: boolean = true + export let contentHeight: number = 0 + let footerHeight: number = 0 + let tableHeight: number = 0 const dispatch = createEventDispatcher() setContext('datatable', { size }) + + $: contentHeight = tableHeight - footerHeight
-
- - -
-
- {#if paginated && !shouldHidePagination} -
-
- {#if rowCount} - {rowCount} items - {/if} -
+ +
+ + +
+
+ {#if paginated && !shouldHidePagination} +
+
+ {#if rowCount} + {rowCount} items + {/if} +
-
- - Page: {currentPage} - {perPage && rowCount ? `/ ${Math.ceil(rowCount / perPage)}` : ''} - +
+ + Page: {currentPage} + {perPage && rowCount ? `/ ${Math.ceil(rowCount / perPage)}` : ''} + - {#if perPage !== undefined} - - {/if} - - {#if showNext} + {#if perPage !== undefined} + + {/if} - {/if} + {#if showNext} + + {/if} +
-
- {:else if shouldLoadMore} -
- -
- {/if} + {:else if shouldLoadMore} +
+ +
+ {/if} +
diff --git a/frontend/src/lib/components/table/Head.svelte b/frontend/src/lib/components/table/Head.svelte index 56882ba412..7dcb13e1f4 100644 --- a/frontend/src/lib/components/table/Head.svelte +++ b/frontend/src/lib/components/table/Head.svelte @@ -1,4 +1,4 @@ - +
diff --git a/frontend/src/lib/components/table/Row.svelte b/frontend/src/lib/components/table/Row.svelte index 19f08bff40..728dde81af 100644 --- a/frontend/src/lib/components/table/Row.svelte +++ b/frontend/src/lib/components/table/Row.svelte @@ -5,6 +5,7 @@ export let hoverable: boolean = false export let selected: boolean = false export let dividable: boolean = false + export let disabled: boolean = false const dispatch = createEventDispatcher() @@ -13,7 +14,8 @@ hoverable ? 'hover:bg-surface-hover cursor-pointer' : '', selected ? 'bg-blue-50 dark:bg-blue-900/50' : '', 'transition-all', - dividable ? 'divide-x' : '' + dividable ? 'divide-x' : '', + disabled ? 'opacity-60' : '' )} on:click={() => { dispatch('click') From c443c2b3a89e070d9fe7388c5da24cbdacab4c10 Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:10:30 +0100 Subject: [PATCH 10/48] nit: format `cache.rs` (#4845) --- backend/windmill-common/src/cache.rs | 177 ++++++++++++++++----------- 1 file changed, 108 insertions(+), 69 deletions(-) diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index bb738bddf1..9810e83be5 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -34,24 +34,36 @@ pub mod flow { /// If not present, import from the file-system cache or fetch it from the database and write /// it to the file system and cache. /// This should be preferred over fetching the database directly. - pub async fn fetch_script(e: impl PgExecutor<'_>, node: FlowNodeId) - -> error::Result<(Option, String)> - { - fetch(e, node).await.and_then(|Val { lock, code, .. }| Ok((lock, code.ok_or_else(|| { - error::Error::InternalErr(format!("Flow node ({:x}) isn't a script node.", node.0)) - })?))) + pub async fn fetch_script( + e: impl PgExecutor<'_>, + node: FlowNodeId, + ) -> error::Result<(Option, String)> { + fetch(e, node).await.and_then(|Val { lock, code, .. }| { + Ok(( + lock, + code.ok_or_else(|| { + error::Error::InternalErr(format!( + "Flow node ({:x}) isn't a script node.", + node.0 + )) + })?, + )) + }) } /// Fetch the flow node flow value referenced by `node` from the cache. /// If not present, import from the file-system cache or fetch it from the database and write /// it to the file system and cache. /// This should be preferred over fetching the database directly. - pub async fn fetch_flow(e: impl PgExecutor<'_>, node: FlowNodeId) - -> error::Result - { - fetch(e, node).await.and_then(|Val { flow, .. }| flow.ok_or_else(|| { - error::Error::InternalErr(format!("Flow node ({:x}) isn't a flow value node.", node.0)) - })) + pub async fn fetch_flow(e: impl PgExecutor<'_>, node: FlowNodeId) -> error::Result { + fetch(e, node).await.and_then(|Val { flow, .. }| { + flow.ok_or_else(|| { + error::Error::InternalErr(format!( + "Flow node ({:x}) isn't a flow value node.", + node.0 + )) + }) + }) } /// Fetch the flow node referenced by `node` from the cache. @@ -62,32 +74,40 @@ pub mod flow { // If not present, `get_or_insert_async` will lock the key until the future completes, // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. - CACHE.get_or_insert_async( - &node, - fs::import_or_insert_with(CACHE_DIR, node.0 as u64, async { - sqlx::query!( - "SELECT \ + CACHE + .get_or_insert_async( + &node, + fs::import_or_insert_with(CACHE_DIR, node.0 as u64, async { + sqlx::query!( + "SELECT \ lock AS \"lock: String\", \ code AS \"code: String\", \ flow::text AS \"flow: Box\" \ FROM flow_node WHERE id = $1 LIMIT 1", - node.0, - ) - .fetch_one(e) - .await - .map_err(Into::into) - .and_then(|r| Ok(Val { - lock: r.lock.and_then(|x| if x.is_empty() { None } else { Some(x) }), - code: r.code, - flow: match r.flow { - None => None, - Some(flow) => serde_json::from_str(&flow).map_err(|err| { - error::Error::InternalErr(format!("Unable to parse flow value: {err:?}")) - })?, - } - })) - }) - ).await + node.0, + ) + .fetch_one(e) + .await + .map_err(Into::into) + .and_then(|r| { + Ok(Val { + lock: r + .lock + .and_then(|x| if x.is_empty() { None } else { Some(x) }), + code: r.code, + flow: match r.flow { + None => None, + Some(flow) => serde_json::from_str(&flow).map_err(|err| { + error::Error::InternalErr(format!( + "Unable to parse flow value: {err:?}" + )) + })?, + }, + }) + }) + }), + ) + .await } // ---------------------------------------------------------------------------------------------- @@ -130,7 +150,11 @@ pub mod flow { match item { Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())), Item::Code => Ok(self.code.as_ref().map(|s| s.as_bytes().to_vec())), - Item::Flow => Ok(self.flow.as_ref().map(|f| serde_json::to_vec(f)).transpose()?), + Item::Flow => Ok(self + .flow + .as_ref() + .map(|f| serde_json::to_vec(f)) + .transpose()?), } } } @@ -164,39 +188,44 @@ pub mod script { /// If not present, import from the file-system cache or fetch it from the database and write /// it to the file system and cache. /// This should be preferred over fetching the database directly. - pub async fn fetch(e: impl PgExecutor<'_>, hash: ScriptHash, workspace_id: &str) - -> error::Result - { + pub async fn fetch( + e: impl PgExecutor<'_>, + hash: ScriptHash, + workspace_id: &str, + ) -> error::Result { // If not present, `get_or_insert_async` will lock the key until the future completes, // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. - CACHE.get_or_insert_async( - &hash, - fs::import_or_insert_with(CACHE_DIR, hash.0 as u64, async { - sqlx::query!( - "SELECT \ + CACHE + .get_or_insert_async( + &hash, + fs::import_or_insert_with(CACHE_DIR, hash.0 as u64, async { + sqlx::query!( + "SELECT \ lock AS \"lock: String\", \ content AS \"code!: String\", language AS \"language: Option\", \ envs AS \"envs: Vec\", \ codebase AS \"codebase: String\" \ FROM script WHERE hash = $1 AND workspace_id = $2 LIMIT 1", - hash.0, - workspace_id, - ) - .fetch_one(e) - .await - .map_err(Into::into) - .map(|r| Val { - lock: r.lock.and_then(|x| if x.is_empty() { None } else { Some(x) }), - code: r.code, - language: r.language, - envs: r.envs, - codebase: r.codebase, - }) - }) - ) - .await + hash.0, + workspace_id, + ) + .fetch_one(e) + .await + .map_err(Into::into) + .map(|r| Val { + lock: r + .lock + .and_then(|x| if x.is_empty() { None } else { Some(x) }), + code: r.code, + language: r.language, + envs: r.envs, + codebase: r.codebase, + }) + }), + ) + .await } // ---------------------------------------------------------------------------------------------- @@ -230,7 +259,9 @@ pub mod script { match item { Item::Lock => self.lock = Some(String::from_utf8(data)?), Item::Code => self.code = String::from_utf8(data)?, - Item::Info => (self.language, self.envs, self.codebase) = serde_json::from_slice(&data)?, + Item::Info => { + (self.language, self.envs, self.codebase) = serde_json::from_slice(&data)? + } } Ok(()) } @@ -239,7 +270,11 @@ pub mod script { match item { Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())), Item::Code => Ok(Some(self.code.as_bytes().to_vec())), - Item::Info => Ok(Some(serde_json::to_vec(&(&self.language, &self.envs, &self.codebase))?)), + Item::Info => Ok(Some(serde_json::to_vec(&( + &self.language, + &self.envs, + &self.codebase, + ))?)), } } } @@ -272,8 +307,7 @@ mod fs { } /// Import or insert a bundle within the given combination of `{root}/{key}/`. - pub async fn import_or_insert_with(root: &str, key: u64, f: F) - -> error::Result + pub async fn import_or_insert_with(root: &str, key: u64, f: F) -> error::Result where T: Bundle, F: Future>, @@ -287,8 +321,9 @@ mod fs { let mut data = T::default(); for item in T::items() { let mut buf = vec![]; - let Ok(mut file) = OpenOptions::new().read(true).open(item.path(&path)) - else { continue }; + let Ok(mut file) = OpenOptions::new().read(true).open(item.path(&path)) else { + continue; + }; file.read_to_end(&mut buf)?; data.import(*item, buf)?; } @@ -299,7 +334,7 @@ mod fs { Ok(data) => return Ok(data), Err(err) => tracing::warn!( "Failed to import from file-system, fetch source..: {path:?}: {err:?}" - ) + ), } } // Cache path doesn't exist or import failed, generate the content. @@ -308,9 +343,13 @@ mod fs { fs::create_dir_all(&path)?; // Write the generated data to the file. for item in T::items() { - let Some(buf) = data.export(*item)? - else { continue }; - let mut file = OpenOptions::new().write(true).create(true).open(item.path(&path))?; + let Some(buf) = data.export(*item)? else { + continue; + }; + let mut file = OpenOptions::new() + .write(true) + .create(true) + .open(item.path(&path))?; file.write_all(&buf)?; } tracing::debug!("Exported to file-system: {:?}", path); From a7bdeb5fcac39112366e363bf32b063f2f1a35a5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 12:36:05 +0100 Subject: [PATCH 11/48] chore(main): release 1.435.0 (#4840) * chore(main): release 1.435.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel --- CHANGELOG.md | 12 +++++++ backend/Cargo.lock | 48 +++++++++++++-------------- backend/Cargo.toml | 4 +-- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +-- frontend/package.json | 2 +- lsp/Pipfile | 4 +-- openflow.openapi.yaml | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 15 files changed, 52 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29628db02c..f9538d8528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.435.0](https://github.com/windmill-labs/windmill/compare/v1.434.2...v1.435.0) (2024-12-05) + + +### Features + +* app custom paths ([#4828](https://github.com/windmill-labs/windmill/issues/4828)) ([1ec6c6f](https://github.com/windmill-labs/windmill/commit/1ec6c6f765904361e641d89495890bc87e8544aa)) + + +### Bug Fixes + +* pass USERPROFILE on windows ([5404ec9](https://github.com/windmill-labs/windmill/commit/5404ec9b48e8d7a0cb27b2319d54f04c94ec6fd0)) + ## [1.434.2](https://github.com/windmill-labs/windmill/compare/v1.434.1...v1.434.2) (2024-12-04) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 93e027e005..e672bff710 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10446,7 +10446,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "axum", @@ -10487,7 +10487,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "argon2", @@ -10573,7 +10573,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.434.2" +version = "1.435.0" dependencies = [ "base64 0.22.1", "chrono", @@ -10591,7 +10591,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.434.2" +version = "1.435.0" dependencies = [ "chrono", "serde", @@ -10604,7 +10604,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "serde", @@ -10618,7 +10618,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "async-stream", @@ -10669,7 +10669,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.434.2" +version = "1.435.0" dependencies = [ "regex", "serde", @@ -10683,7 +10683,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "bytes", @@ -10706,7 +10706,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.434.2" +version = "1.435.0" dependencies = [ "itertools 0.13.0", "lazy_static", @@ -10718,7 +10718,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.434.2" +version = "1.435.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -10727,7 +10727,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "lazy_static", @@ -10739,7 +10739,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "gosyn", @@ -10751,7 +10751,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "lazy_static", @@ -10763,7 +10763,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10774,7 +10774,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10785,7 +10785,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "async-recursion", @@ -10803,7 +10803,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10820,7 +10820,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "lazy_static", @@ -10832,7 +10832,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "lazy_static", @@ -10850,7 +10850,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10871,7 +10871,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "serde_json", @@ -10881,7 +10881,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "async-recursion", @@ -10915,7 +10915,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.434.2" +version = "1.435.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10925,7 +10925,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.434.2" +version = "1.435.0" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6b70463727..efedca1e30 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.434.2" +version = "1.435.0" authors.workspace = true edition.workspace = true @@ -29,7 +29,7 @@ members = [ ] [workspace.package] -version = "1.434.2" +version = "1.435.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4ec45bbdfa..5d738cc1f4 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.434.2 + version: 1.435.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index da88c5d0aa..93ccba9355 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.434.2"; +export const VERSION = "v1.435.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index bf465fc315..e8da61ca9c 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.434.2"; +export const VERSION = "1.435.0"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 237bcb81bd..331d8e0754 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.434.2", + "version": "1.435.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.434.2", + "version": "1.435.0", "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.32.1", diff --git a/frontend/package.json b/frontend/package.json index 24cf283420..cd7e8ea13c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.434.2", + "version": "1.435.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index b65209dcd2..7b74af871b 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.434.2" -wmill_pg = ">=1.434.2" +wmill = ">=1.435.0" +wmill_pg = ">=1.435.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 9ed0fede12..220f6b30f0 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.434.2 + version: 1.435.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 1ee961678d..dc4cc79cfb 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.434.2" +version = "1.435.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 4e41543428..a898c49cf5 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.434.2" +version = "1.435.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index e113d36b82..9376e07041 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.434.2", + "version": "1.435.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index ac77f8302b..418eea95d1 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.434.2", + "version": "1.435.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 417684d0b6..f14dc64823 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.434.2 +1.435.0 From 548cfcfbde23ac7ae129e10f6e92d57516e61f20 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 13:00:17 +0100 Subject: [PATCH 12/48] fix: improve critical alerts filters --- .../src/lib/components/sidebar/CriticalAlertModal.svelte | 5 +++-- .../lib/components/sidebar/CriticalAlertModalInner.svelte | 2 +- powershell-client/WindmillClient/WindmillClient.psd1 | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte b/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte index 94675fba0a..399e044ee5 100644 --- a/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte +++ b/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte @@ -20,9 +20,9 @@ export let open: boolean = false export let numUnacknowledgedCriticalAlerts: number = 0 - export let muteSettings; + export let muteSettings let workspaceContext = false - let childRef; + let childRef $: { setupApiFunctions(workspaceContext) @@ -275,6 +275,7 @@ Date: Thu, 5 Dec 2024 13:36:12 +0100 Subject: [PATCH 13/48] Added space after Use simplified builder button (#4848) --- frontend/src/lib/components/CronInput.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/CronInput.svelte b/frontend/src/lib/components/CronInput.svelte index 3ce3caf385..039cc9409a 100644 --- a/frontend/src/lib/components/CronInput.svelte +++ b/frontend/src/lib/components/CronInput.svelte @@ -250,7 +250,7 @@ {#if !disabled} -
+
From 93319f45bb5dec01de553f53c2cf6a1f7a949ee8 Mon Sep 17 00:00:00 2001 From: Henri Courdent <122811744+hcourdent@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:36:23 +0100 Subject: [PATCH 14/48] Fix dead links (#4847) --- frontend/src/lib/components/InstanceSettings.svelte | 2 +- frontend/src/lib/components/ScriptBuilder.svelte | 2 +- frontend/src/lib/components/flows/content/FlowInputs.svelte | 2 +- frontend/src/lib/components/sidebar/changelogs.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 0b509b0814..ea8f9bf889 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -325,7 +325,7 @@
Setting SMTP unlocks sending emails upon adding new users to the workspace or the instance or sending critical alerts. - Learn moreLearn more
{:else if category == "Indexer/Search"} diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 4728ae8294..3b9fddf6cf 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -648,7 +648,7 @@ Triggers Configure how this script will be triggered. diff --git a/frontend/src/lib/components/flows/content/FlowInputs.svelte b/frontend/src/lib/components/flows/content/FlowInputs.svelte index fb27580ae5..65c8b137f6 100644 --- a/frontend/src/lib/components/flows/content/FlowInputs.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputs.svelte @@ -121,7 +121,7 @@ By default, adding a trigger will set the schedule to 15 minutes. To see all ways to trigger a flow, check Triggering Flows. diff --git a/frontend/src/lib/components/sidebar/changelogs.ts b/frontend/src/lib/components/sidebar/changelogs.ts index 8de6be0fe4..0c6be5542e 100644 --- a/frontend/src/lib/components/sidebar/changelogs.ts +++ b/frontend/src/lib/components/sidebar/changelogs.ts @@ -22,7 +22,7 @@ const changelogs: Changelog[] = [ }, { label: 'Critical alert channels', - href: 'https://www.windmill.dev/changelog/critical-alert-channels', + href: 'https://www.windmill.dev/changelog/critical-alerts', date: '2024-09-01' }, { From 7284f72427019f828723ee546eb15064239ca1f0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 13:37:44 +0100 Subject: [PATCH 15/48] chore(main): release 1.435.1 (#4846) * chore(main): release 1.435.1 * Apply automatic changes * Update WindmillClient.psd1 --------- Co-authored-by: rubenfiszel --- CHANGELOG.md | 7 +++ backend/Cargo.lock | 48 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 48 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9538d8528..e305c73393 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.435.1](https://github.com/windmill-labs/windmill/compare/v1.435.0...v1.435.1) (2024-12-05) + + +### Bug Fixes + +* improve critical alerts filters ([548cfcf](https://github.com/windmill-labs/windmill/commit/548cfcfbde23ac7ae129e10f6e92d57516e61f20)) + ## [1.435.0](https://github.com/windmill-labs/windmill/compare/v1.434.2...v1.435.0) (2024-12-05) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e672bff710..54758013bc 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10446,7 +10446,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "axum", @@ -10487,7 +10487,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "argon2", @@ -10573,7 +10573,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.435.0" +version = "1.435.1" dependencies = [ "base64 0.22.1", "chrono", @@ -10591,7 +10591,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.435.0" +version = "1.435.1" dependencies = [ "chrono", "serde", @@ -10604,7 +10604,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "serde", @@ -10618,7 +10618,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "async-stream", @@ -10669,7 +10669,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.435.0" +version = "1.435.1" dependencies = [ "regex", "serde", @@ -10683,7 +10683,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "bytes", @@ -10706,7 +10706,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.435.0" +version = "1.435.1" dependencies = [ "itertools 0.13.0", "lazy_static", @@ -10718,7 +10718,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.435.0" +version = "1.435.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -10727,7 +10727,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "lazy_static", @@ -10739,7 +10739,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "gosyn", @@ -10751,7 +10751,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "lazy_static", @@ -10763,7 +10763,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10774,7 +10774,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10785,7 +10785,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "async-recursion", @@ -10803,7 +10803,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10820,7 +10820,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "lazy_static", @@ -10832,7 +10832,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "lazy_static", @@ -10850,7 +10850,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10871,7 +10871,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "serde_json", @@ -10881,7 +10881,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "async-recursion", @@ -10915,7 +10915,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.435.0" +version = "1.435.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10925,7 +10925,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.435.0" +version = "1.435.1" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index efedca1e30..6fdf37998c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.435.0" +version = "1.435.1" authors.workspace = true edition.workspace = true @@ -29,7 +29,7 @@ members = [ ] [workspace.package] -version = "1.435.0" +version = "1.435.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 5d738cc1f4..089f330cbb 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.435.0 + version: 1.435.1 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 93ccba9355..86c2e9535f 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.435.0"; +export const VERSION = "v1.435.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index e8da61ca9c..632047531b 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.435.0"; +export const VERSION = "1.435.1"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 331d8e0754..13b3a8612a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.435.0", + "version": "1.435.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.435.0", + "version": "1.435.1", "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.32.1", diff --git a/frontend/package.json b/frontend/package.json index cd7e8ea13c..260d45e079 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.435.0", + "version": "1.435.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 7b74af871b..40b1f136b3 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.435.0" -wmill_pg = ">=1.435.0" +wmill = ">=1.435.1" +wmill_pg = ">=1.435.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 220f6b30f0..281b578ecc 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.435.0 + version: 1.435.1 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index fd2c09cb69..f31c6ac7f2 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.435.0' + ModuleVersion = '1.435.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index dc4cc79cfb..434f94a351 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.435.0" +version = "1.435.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index a898c49cf5..ce73862ae4 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.435.0" +version = "1.435.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 9376e07041..d90bf312b9 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.435.0", + "version": "1.435.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 418eea95d1..acaf6a59dc 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.435.0", + "version": "1.435.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index f14dc64823..4e989d00fa 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.435.0 +1.435.1 From 2c934cc3b88e3838d134b62c6be0f7bdbc4406bb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 14:45:23 +0100 Subject: [PATCH 16/48] improve windows worker --- backend/windmill-worker/src/worker.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b47f63e4c3..47f2fb393f 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -422,10 +422,6 @@ lazy_static::lazy_static! { .and_then(|x| x.parse().ok()) .unwrap_or(false); - pub static ref USERPROFILE_ENV: String = std::env::var("USERPROFILE").unwrap_or_else(|_| "/tmp".to_string()); - - - } #[cfg(windows)] From 85df359c0a3ec0f790ee320e2d70d04b91dbe9e5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 15:01:47 +0100 Subject: [PATCH 17/48] improve handling of unexpected toast errors --- frontend/src/lib/components/toast.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/toast.ts b/frontend/src/lib/components/toast.ts index f1f1a84f19..7290df87e5 100644 --- a/frontend/src/lib/components/toast.ts +++ b/frontend/src/lib/components/toast.ts @@ -2,7 +2,11 @@ const pathRegex = /\b(u|f)\/[^\/\s]+\/[^\/\s]+\b/g export function processMessage(message: string | undefined): string { - return (message ?? 'Error without message').replaceAll(pathRegex, (path) => { + return ( + typeof message == 'string' + ? message ?? 'Error without message' + : JSON.stringify(message, null, 2) + ).replaceAll(pathRegex, (path) => { return `${path}` }) } From b19faa45cd37822874d907abe9ded8eb8b20557b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 15:17:24 +0100 Subject: [PATCH 18/48] fix compile --- backend/windmill-worker/src/rust_executor.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 3f5ff3d18e..c6f258d6ac 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -403,8 +403,10 @@ pub async fn handle_rust_job( .stderr(Stdio::piped()); #[cfg(windows)] - run_rust.env("SystemRoot", SYSTEM_ROOT.as_str()); - run_rust.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + { + run_rust.env("SystemRoot", SYSTEM_ROOT.as_str()); + run_rust.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + } start_child_process(run_rust, compiled_executable_name).await? }; From 96d4af0254aeb067c0120e7fc7554c79033d36d8 Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:17:41 +0100 Subject: [PATCH 19/48] nit: format `worker_lockfiles.rs` (#4849) --- .../windmill-worker/src/worker_lockfiles.rs | 127 +++++++++++++----- 1 file changed, 95 insertions(+), 32 deletions(-) diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 552e15dce5..71834a9f04 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -606,9 +606,8 @@ pub async fn handle_flow_dependency_job( occupancy_metrics, ) .await?; - let new_flow_value = sqlx::types::Json( - serde_json::value::to_raw_value(&flow).map_err(to_anyhow)? - ); + let new_flow_value = + sqlx::types::Json(serde_json::value::to_raw_value(&flow).map_err(to_anyhow)?); // Re-check cancelation to ensure we don't accidentially override a flow. if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id) @@ -648,11 +647,20 @@ pub async fn handle_flow_dependency_job( // Compute a lite version of the flow value (`RawScript` => `FlowScript`). let mut value_lite = flow.clone(); - tx = reduce(tx, &mut value_lite.modules, &job_path, &job.workspace_id, flow.failure_module.as_ref(), flow.same_worker).await?; + tx = reduce( + tx, + &mut value_lite.modules, + &job_path, + &job.workspace_id, + flow.failure_module.as_ref(), + flow.same_worker, + ) + .await?; sqlx::query!( "INSERT INTO flow_version_lite (id, value) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value", - version, sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json>, + version, + sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json>, ) .execute(db) .await?; @@ -829,8 +837,12 @@ async fn lock_modules<'c>( occupancy_metrics, )) .await?; - e.value = - FlowModuleValue::WhileloopFlow { modules: nmodules, modules_node, skip_failures }.into() + e.value = FlowModuleValue::WhileloopFlow { + modules: nmodules, + modules_node, + skip_failures, + } + .into() } FlowModuleValue::BranchOne { branches, default, default_node } => { let mut nbranches = vec![]; @@ -878,8 +890,12 @@ async fn lock_modules<'c>( occupancy_metrics, )) .await?; - e.value = FlowModuleValue::BranchOne { branches: nbranches, default: ndefault, default_node } - .into(); + e.value = FlowModuleValue::BranchOne { + branches: nbranches, + default: ndefault, + default_node, + } + .into(); } _ => (), }; @@ -1009,8 +1025,8 @@ async fn insert_flow_node<'c>( flow: Option<&Json>>, ) -> Result<(sqlx::Transaction<'c, sqlx::Postgres>, FlowNodeId)> { let hash = { - use std::hash::{DefaultHasher, Hasher, Hash}; - + use std::hash::{DefaultHasher, Hash, Hasher}; + let mut hasher = DefaultHasher::new(); code.hash(&mut hasher); lock.hash(&mut hasher); @@ -1039,7 +1055,12 @@ async fn insert_flow_node<'c>( UNION ALL SELECT id FROM inserted "#, - hash, path, workspace_id, code, lock, flow as Option<&Json>> + hash, + path, + workspace_id, + code, + lock, + flow as Option<&Json>> ) .fetch_one(&mut *tx) .await? @@ -1056,7 +1077,15 @@ async fn insert_flow_modules<'c>( modules: &mut Vec, modules_node: &mut Option, ) -> Result> { - tx = Box::pin(reduce(tx, modules, path, workspace_id, failure_module, same_worker)).await?; + tx = Box::pin(reduce( + tx, + modules, + path, + workspace_id, + failure_module, + same_worker, + )) + .await?; add_virtual_items_if_necessary(modules); if modules.is_empty() || crate::worker_flow::is_simple_modules(modules, failure_module) { return Ok(tx); @@ -1073,7 +1102,7 @@ async fn insert_flow_modules<'c>( failure_module: failure_module.cloned(), same_worker, ..Default::default() - }))) + }))), ) .await?; *modules_node = Some(id); @@ -1090,8 +1119,13 @@ async fn reduce<'c>( ) -> Result> { use FlowModuleValue::*; for module in &mut *modules { - let mut val = serde_json::from_str::(module.value.get()) - .map_err(|err| Error::InternalErr(format!("reduce: Failed to parse flow module value: {}", err)))?; + let mut val = + serde_json::from_str::(module.value.get()).map_err(|err| { + Error::InternalErr(format!( + "reduce: Failed to parse flow module value: {}", + err + )) + })?; match &mut val { RawScript { .. } => { // In order to avoid an unnecessary `.clone()` of `val`, take ownership of it's content @@ -1107,9 +1141,14 @@ async fn reduce<'c>( concurrency_time_window_s, is_trigger, .. - } = std::mem::replace(&mut val, Identity) else { unreachable!() }; + } = std::mem::replace(&mut val, Identity) + else { + unreachable!() + }; let id; - (tx, id) = insert_flow_node(tx, path, workspace_id, Some(&content), lock.as_ref(), None).await?; + (tx, id) = + insert_flow_node(tx, path, workspace_id, Some(&content), lock.as_ref(), None) + .await?; val = FlowScript { input_transforms, id, @@ -1120,32 +1159,56 @@ async fn reduce<'c>( concurrency_time_window_s, is_trigger, }; - }, + } ForloopFlow { modules, modules_node, .. } - | WhileloopFlow { modules, modules_node, .. } => { + | WhileloopFlow { modules, modules_node, .. } => { tx = insert_flow_modules( - tx, path, workspace_id, failure_module, same_worker, - modules, modules_node - ).await?; + tx, + path, + workspace_id, + failure_module, + same_worker, + modules, + modules_node, + ) + .await?; } BranchOne { branches, default, default_node, .. } => { for branch in branches.iter_mut() { tx = insert_flow_modules( - tx, path, workspace_id, failure_module, same_worker, - &mut branch.modules, &mut branch.modules_node - ).await?; + tx, + path, + workspace_id, + failure_module, + same_worker, + &mut branch.modules, + &mut branch.modules_node, + ) + .await?; } tx = insert_flow_modules( - tx, path, workspace_id, failure_module, same_worker, - default, default_node - ).await?; + tx, + path, + workspace_id, + failure_module, + same_worker, + default, + default_node, + ) + .await?; } BranchAll { branches, .. } => { for branch in branches.iter_mut() { tx = insert_flow_modules( - tx, path, workspace_id, failure_module, same_worker, - &mut branch.modules, &mut branch.modules_node - ).await?; + tx, + path, + workspace_id, + failure_module, + same_worker, + &mut branch.modules, + &mut branch.modules_node, + ) + .await?; } } _ => {} From a99e63f5435725c42e38b46933ff3caa345002b5 Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:10:59 +0100 Subject: [PATCH 20/48] fix: job search toast on error (#4851) --- frontend/src/lib/components/search/GlobalSearchModal.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index c443a55e8f..d76a25f96c 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -244,7 +244,7 @@ queryParseErrors = searchResults.query_parse_errors indexMetadata = searchResults.index_metadata } catch (e) { - sendUserToast(e, true) + sendUserToast(e.body, true) } loadingCompletedRuns = false selectedItem = selectItem(0) From c67fd8dbb904071e5bfe6483f51a49f48e7c7c02 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 5 Dec 2024 10:19:59 -0500 Subject: [PATCH 21/48] removing unneccessary requests to backend (#4852) --- .../sidebar/CriticalAlertModalInner.svelte | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte b/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte index 365451a334..6127475cfd 100644 --- a/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte +++ b/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte @@ -2,7 +2,6 @@ import Toggle from '$lib/components/Toggle.svelte' import { SettingService } from '$lib/gen' import type { CriticalAlert } from '$lib/gen' - import { onMount } from 'svelte' import { devopsRole, instanceSettingsSelectedTab, superadmin } from '$lib/stores' import { goto } from '$app/navigation' import List from '$lib/components/common/layout/List.svelte' @@ -27,10 +26,6 @@ refreshAlerts() } - onMount(() => { - refreshAlerts() - }) - // Pagination let page = 1 let pageSize = 10 @@ -54,6 +49,7 @@ }) hasMore = pageNumber < res.total_pages + totalNumberOfAlerts = res.total_rows filteredAlerts = res.alerts updateHasUnacknowledgedCriticalAlerts() } finally { @@ -67,8 +63,6 @@ if (reset) { page = 1 } - updateHasUnacknowledgedCriticalAlerts() - await getTotalNumber() await fetchAlerts(page) } @@ -108,23 +102,12 @@ function onFiltersChange() { getAlerts(true) - getTotalNumber() } // Update filter change handlers $: hideAcknowledged, workspaceContext, onFiltersChange() let totalNumberOfAlerts = 0 - async function getTotalNumber() { - loading = true - const res = await getCriticalAlerts({ - page: 1, - pageSize: 1000, - acknowledged: hideAcknowledged ? false : undefined - }) - totalNumberOfAlerts = res.total_rows - loading = false - } From 185848ceb184e86d307aaf6f78c0d72ae1e26644 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 16:28:00 +0100 Subject: [PATCH 22/48] chore(main): release 1.435.2 (#4853) * chore(main): release 1.435.2 * Apply automatic changes * Update WindmillClient.psd1 --------- Co-authored-by: rubenfiszel --- CHANGELOG.md | 7 +++ backend/Cargo.lock | 48 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 48 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e305c73393..94c2ef5cf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.435.2](https://github.com/windmill-labs/windmill/compare/v1.435.1...v1.435.2) (2024-12-05) + + +### Bug Fixes + +* job search toast on error ([#4851](https://github.com/windmill-labs/windmill/issues/4851)) ([a99e63f](https://github.com/windmill-labs/windmill/commit/a99e63f5435725c42e38b46933ff3caa345002b5)) + ## [1.435.1](https://github.com/windmill-labs/windmill/compare/v1.435.0...v1.435.1) (2024-12-05) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 54758013bc..dac85d4f09 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10446,7 +10446,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "axum", @@ -10487,7 +10487,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "argon2", @@ -10573,7 +10573,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.435.1" +version = "1.435.2" dependencies = [ "base64 0.22.1", "chrono", @@ -10591,7 +10591,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.435.1" +version = "1.435.2" dependencies = [ "chrono", "serde", @@ -10604,7 +10604,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "serde", @@ -10618,7 +10618,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "async-stream", @@ -10669,7 +10669,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.435.1" +version = "1.435.2" dependencies = [ "regex", "serde", @@ -10683,7 +10683,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "bytes", @@ -10706,7 +10706,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.435.1" +version = "1.435.2" dependencies = [ "itertools 0.13.0", "lazy_static", @@ -10718,7 +10718,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.435.1" +version = "1.435.2" dependencies = [ "convert_case 0.6.0", "serde", @@ -10727,7 +10727,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "lazy_static", @@ -10739,7 +10739,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "gosyn", @@ -10751,7 +10751,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "lazy_static", @@ -10763,7 +10763,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10774,7 +10774,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10785,7 +10785,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "async-recursion", @@ -10803,7 +10803,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10820,7 +10820,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "lazy_static", @@ -10832,7 +10832,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "lazy_static", @@ -10850,7 +10850,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10871,7 +10871,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "serde_json", @@ -10881,7 +10881,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "async-recursion", @@ -10915,7 +10915,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.435.1" +version = "1.435.2" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10925,7 +10925,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.435.1" +version = "1.435.2" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6fdf37998c..3f591dfd84 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.435.1" +version = "1.435.2" authors.workspace = true edition.workspace = true @@ -29,7 +29,7 @@ members = [ ] [workspace.package] -version = "1.435.1" +version = "1.435.2" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 089f330cbb..95c06d5aae 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.435.1 + version: 1.435.2 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 86c2e9535f..2848c702dc 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.435.1"; +export const VERSION = "v1.435.2"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 632047531b..1bad532ef9 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.435.1"; +export const VERSION = "1.435.2"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 13b3a8612a..024b5e634c 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.435.1", + "version": "1.435.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.435.1", + "version": "1.435.2", "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.32.1", diff --git a/frontend/package.json b/frontend/package.json index 260d45e079..847f5b4ee7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.435.1", + "version": "1.435.2", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 40b1f136b3..45bf361b11 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.435.1" -wmill_pg = ">=1.435.1" +wmill = ">=1.435.2" +wmill_pg = ">=1.435.2" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 281b578ecc..058620949e 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.435.1 + version: 1.435.2 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index f31c6ac7f2..89cf53b4af 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.435.1' + ModuleVersion = '1.435.2' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 434f94a351..14008078cc 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.435.1" +version = "1.435.2" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index ce73862ae4..9111d7cfb1 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.435.1" +version = "1.435.2" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index d90bf312b9..0dfcc51a4a 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.435.1", + "version": "1.435.2", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index acaf6a59dc..f351d3ae55 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.435.1", + "version": "1.435.2", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 4e989d00fa..5b445a69a9 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.435.1 +1.435.2 From cf7278e2457c641ce96697f29ebfabefde9b20bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:40:35 +0100 Subject: [PATCH 23/48] chore(deps-dev): bump @sveltejs/kit from 2.5.0 to 2.9.0 in /frontend (#4832) Bumps [@sveltejs/kit](https://github.com/sveltejs/kit/tree/HEAD/packages/kit) from 2.5.0 to 2.9.0. - [Release notes](https://github.com/sveltejs/kit/releases) - [Changelog](https://github.com/sveltejs/kit/blob/main/packages/kit/CHANGELOG.md) - [Commits](https://github.com/sveltejs/kit/commits/@sveltejs/kit@2.9.0/packages/kit) --- updated-dependencies: - dependency-name: "@sveltejs/kit" dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ruben Fiszel --- frontend/package-lock.json | 66 +++++++++++++++++++++----------------- frontend/package.json | 2 +- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 024b5e634c..0e2fb04480 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -26,7 +26,7 @@ "@popperjs/core": "^2.11.6", "@redocly/json-to-json-schema": "^0.0.1", "@tanstack/svelte-table": "^8.9.9", - "@windmill-labs/svelte-dnd-action": "latest", + "@windmill-labs/svelte-dnd-action": "*", "@xyflow/svelte": "^0.1.15", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", @@ -88,7 +88,7 @@ "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", "@sveltejs/adapter-static": "^3.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/kit": "^2.9.0", "@sveltejs/package": "^2.2.2", "@sveltejs/vite-plugin-svelte": "^3.0.0", "@tailwindcss/forms": "^0.5.3", @@ -3897,10 +3897,11 @@ } }, "node_modules/@polka/url": { - "version": "1.0.0-next.24", - "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.24.tgz", - "integrity": "sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ==", - "dev": true + "version": "1.0.0-next.28", + "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.28.tgz", + "integrity": "sha512-8LduaNlMZGwdZ6qWrKlfa+2M4gahzFkprZiAt2TF8uS0qQgBizKXpXURqvTJ4WtmupWxaLqjRb2UCTe72mu+Aw==", + "dev": true, + "license": "MIT" }, "node_modules/@popperjs/core": { "version": "2.11.8", @@ -4223,23 +4224,24 @@ } }, "node_modules/@sveltejs/kit": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.5.0.tgz", - "integrity": "sha512-1uyXvzC2Lu1FZa30T4y5jUAC21R309ZMRG0TPt+PPPbNUoDpy8zSmSNVWYaBWxYDqLGQ5oPNWvjvvF2IjJ1jmA==", + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.9.0.tgz", + "integrity": "sha512-W3E7ed3ChB6kPqRs2H7tcHp+Z7oiTFC6m+lLyAQQuyXeqw6LdNuuwEUla+5VM0OGgqQD+cYD6+7Xq80vVm17Vg==", "dev": true, "hasInstallScript": true, + "license": "MIT", "dependencies": { "@types/cookie": "^0.6.0", "cookie": "^0.6.0", - "devalue": "^4.3.2", - "esm-env": "^1.0.0", - "import-meta-resolve": "^4.0.0", + "devalue": "^5.1.0", + "esm-env": "^1.2.1", + "import-meta-resolve": "^4.1.0", "kleur": "^4.1.5", "magic-string": "^0.30.5", "mrmime": "^2.0.0", "sade": "^1.8.1", "set-cookie-parser": "^2.6.0", - "sirv": "^2.0.4", + "sirv": "^3.0.0", "tiny-glob": "^0.2.9" }, "bin": { @@ -4249,9 +4251,9 @@ "node": ">=18.13" }, "peerDependencies": { - "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0", "svelte": "^4.0.0 || ^5.0.0-next.0", - "vite": "^5.0.3" + "vite": "^5.0.3 || ^6.0.0" } }, "node_modules/@sveltejs/package": { @@ -6434,10 +6436,11 @@ } }, "node_modules/devalue": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/devalue/-/devalue-4.3.2.tgz", - "integrity": "sha512-KqFl6pOgOW+Y6wJgu80rHpo2/3H07vr8ntR9rkkFIRETewbf5GaYYcakYfiKz89K+sLsuPkQIZaXDMjUObZwWg==", - "dev": true + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.1.1.tgz", + "integrity": "sha512-maua5KUiapvEwiEAe+XnlZ3Rh0GD+qI1J/nb9vrJc3muPXvcF/8gXYTWF76+5DAqHyDUtOIImEuo0YKE9mshVw==", + "dev": true, + "license": "MIT" }, "node_modules/devlop": { "version": "1.1.0", @@ -6990,9 +6993,10 @@ } }, "node_modules/esm-env": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.0.0.tgz", - "integrity": "sha512-Cf6VksWPsTuW01vU9Mk/3vRue91Zevka5SjyNf3nEpokFRuqt/KjUQoGAwq9qMmhpLTHmXzSIrFRw8zxWzmFBA==" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.1.tgz", + "integrity": "sha512-U9JedYYjCnadUlXk7e1Kr+aENQhtUaoaV9+gZm1T8LC/YBAPJx3NSPIAurFOC0U5vrdSevnUJS2/wUVxGwPhng==", + "license": "MIT" }, "node_modules/esm-env-robust": { "version": "0.0.3", @@ -7932,10 +7936,11 @@ } }, "node_modules/import-meta-resolve": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.0.0.tgz", - "integrity": "sha512-okYUR7ZQPH+efeuMJGlq4f8ubUgO50kByRPyt/Cy1Io4PSRsPjxME+YlVaCOx+NIToW7hCsZNFJyTPFFKepRSA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz", + "integrity": "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==", "dev": true, + "license": "MIT", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -9781,6 +9786,7 @@ "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.0.tgz", "integrity": "sha512-eu38+hdgojoyq63s+yTpN4XMBdt5l8HhMhc4VKLO9KM5caLIBvUm4thi7fFaxyTmCKeNnXZ5pAlBwCUnhA09uw==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" } @@ -11960,17 +11966,18 @@ "dev": true }, "node_modules/sirv": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/sirv/-/sirv-2.0.4.tgz", - "integrity": "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.0.tgz", + "integrity": "sha512-BPwJGUeDaDCHihkORDchNyyTvWFhcusy1XMmhEVTQTwGeybFbp8YEmB+njbPnth1FibULBSBVwCQni25XlCUDg==", "dev": true, + "license": "MIT", "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", "totalist": "^3.0.0" }, "engines": { - "node": ">= 10" + "node": ">=18" } }, "node_modules/slash": { @@ -13064,6 +13071,7 @@ "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz", "integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } diff --git a/frontend/package.json b/frontend/package.json index 847f5b4ee7..15ccede702 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -22,7 +22,7 @@ "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", "@sveltejs/adapter-static": "^3.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/kit": "^2.9.0", "@sveltejs/package": "^2.2.2", "@sveltejs/vite-plugin-svelte": "^3.0.0", "@tailwindcss/forms": "^0.5.3", From 205c1a69f28d6a51a9575bfc258a801b3a895e5d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 16:42:04 +0100 Subject: [PATCH 24/48] =?UTF-8?q?Revert=20"chore(deps-dev):=20bump=20@svel?= =?UTF-8?q?tejs/kit=20from=202.5.0=20to=202.9.0=20in=20/frontend=20?= =?UTF-8?q?=E2=80=A6"=20(#4854)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit cf7278e2457c641ce96697f29ebfabefde9b20bf. --- frontend/package-lock.json | 66 +++++++++++++++++--------------------- frontend/package.json | 2 +- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0e2fb04480..024b5e634c 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -26,7 +26,7 @@ "@popperjs/core": "^2.11.6", "@redocly/json-to-json-schema": "^0.0.1", "@tanstack/svelte-table": "^8.9.9", - "@windmill-labs/svelte-dnd-action": "*", + "@windmill-labs/svelte-dnd-action": "latest", "@xyflow/svelte": "^0.1.15", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", @@ -88,7 +88,7 @@ "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", "@sveltejs/adapter-static": "^3.0.0", - "@sveltejs/kit": "^2.9.0", + "@sveltejs/kit": "^2.0.0", "@sveltejs/package": "^2.2.2", "@sveltejs/vite-plugin-svelte": "^3.0.0", "@tailwindcss/forms": "^0.5.3", @@ -3897,11 +3897,10 @@ } }, "node_modules/@polka/url": { - "version": "1.0.0-next.28", - "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.28.tgz", - "integrity": "sha512-8LduaNlMZGwdZ6qWrKlfa+2M4gahzFkprZiAt2TF8uS0qQgBizKXpXURqvTJ4WtmupWxaLqjRb2UCTe72mu+Aw==", - "dev": true, - "license": "MIT" + "version": "1.0.0-next.24", + "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.24.tgz", + "integrity": "sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ==", + "dev": true }, "node_modules/@popperjs/core": { "version": "2.11.8", @@ -4224,24 +4223,23 @@ } }, "node_modules/@sveltejs/kit": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.9.0.tgz", - "integrity": "sha512-W3E7ed3ChB6kPqRs2H7tcHp+Z7oiTFC6m+lLyAQQuyXeqw6LdNuuwEUla+5VM0OGgqQD+cYD6+7Xq80vVm17Vg==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.5.0.tgz", + "integrity": "sha512-1uyXvzC2Lu1FZa30T4y5jUAC21R309ZMRG0TPt+PPPbNUoDpy8zSmSNVWYaBWxYDqLGQ5oPNWvjvvF2IjJ1jmA==", "dev": true, "hasInstallScript": true, - "license": "MIT", "dependencies": { "@types/cookie": "^0.6.0", "cookie": "^0.6.0", - "devalue": "^5.1.0", - "esm-env": "^1.2.1", - "import-meta-resolve": "^4.1.0", + "devalue": "^4.3.2", + "esm-env": "^1.0.0", + "import-meta-resolve": "^4.0.0", "kleur": "^4.1.5", "magic-string": "^0.30.5", "mrmime": "^2.0.0", "sade": "^1.8.1", "set-cookie-parser": "^2.6.0", - "sirv": "^3.0.0", + "sirv": "^2.0.4", "tiny-glob": "^0.2.9" }, "bin": { @@ -4251,9 +4249,9 @@ "node": ">=18.13" }, "peerDependencies": { - "@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0", + "@sveltejs/vite-plugin-svelte": "^3.0.0", "svelte": "^4.0.0 || ^5.0.0-next.0", - "vite": "^5.0.3 || ^6.0.0" + "vite": "^5.0.3" } }, "node_modules/@sveltejs/package": { @@ -6436,11 +6434,10 @@ } }, "node_modules/devalue": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.1.1.tgz", - "integrity": "sha512-maua5KUiapvEwiEAe+XnlZ3Rh0GD+qI1J/nb9vrJc3muPXvcF/8gXYTWF76+5DAqHyDUtOIImEuo0YKE9mshVw==", - "dev": true, - "license": "MIT" + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-4.3.2.tgz", + "integrity": "sha512-KqFl6pOgOW+Y6wJgu80rHpo2/3H07vr8ntR9rkkFIRETewbf5GaYYcakYfiKz89K+sLsuPkQIZaXDMjUObZwWg==", + "dev": true }, "node_modules/devlop": { "version": "1.1.0", @@ -6993,10 +6990,9 @@ } }, "node_modules/esm-env": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.1.tgz", - "integrity": "sha512-U9JedYYjCnadUlXk7e1Kr+aENQhtUaoaV9+gZm1T8LC/YBAPJx3NSPIAurFOC0U5vrdSevnUJS2/wUVxGwPhng==", - "license": "MIT" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.0.0.tgz", + "integrity": "sha512-Cf6VksWPsTuW01vU9Mk/3vRue91Zevka5SjyNf3nEpokFRuqt/KjUQoGAwq9qMmhpLTHmXzSIrFRw8zxWzmFBA==" }, "node_modules/esm-env-robust": { "version": "0.0.3", @@ -7936,11 +7932,10 @@ } }, "node_modules/import-meta-resolve": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz", - "integrity": "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.0.0.tgz", + "integrity": "sha512-okYUR7ZQPH+efeuMJGlq4f8ubUgO50kByRPyt/Cy1Io4PSRsPjxME+YlVaCOx+NIToW7hCsZNFJyTPFFKepRSA==", "dev": true, - "license": "MIT", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -9786,7 +9781,6 @@ "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.0.tgz", "integrity": "sha512-eu38+hdgojoyq63s+yTpN4XMBdt5l8HhMhc4VKLO9KM5caLIBvUm4thi7fFaxyTmCKeNnXZ5pAlBwCUnhA09uw==", "dev": true, - "license": "MIT", "engines": { "node": ">=10" } @@ -11966,18 +11960,17 @@ "dev": true }, "node_modules/sirv": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.0.tgz", - "integrity": "sha512-BPwJGUeDaDCHihkORDchNyyTvWFhcusy1XMmhEVTQTwGeybFbp8YEmB+njbPnth1FibULBSBVwCQni25XlCUDg==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/sirv/-/sirv-2.0.4.tgz", + "integrity": "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==", "dev": true, - "license": "MIT", "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", "totalist": "^3.0.0" }, "engines": { - "node": ">=18" + "node": ">= 10" } }, "node_modules/slash": { @@ -13071,7 +13064,6 @@ "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz", "integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==", "dev": true, - "license": "MIT", "engines": { "node": ">=6" } diff --git a/frontend/package.json b/frontend/package.json index 15ccede702..847f5b4ee7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -22,7 +22,7 @@ "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", "@sveltejs/adapter-static": "^3.0.0", - "@sveltejs/kit": "^2.9.0", + "@sveltejs/kit": "^2.0.0", "@sveltejs/package": "^2.2.2", "@sveltejs/vite-plugin-svelte": "^3.0.0", "@tailwindcss/forms": "^0.5.3", From c4163aabcd384817b9d4ec1017c257f5da7fadf6 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 5 Dec 2024 11:00:20 -0500 Subject: [PATCH 25/48] update version update script to match new pwsh module formatting (#4855) --- .github/change-versions-mac.sh | 2 +- .github/change-versions.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/change-versions-mac.sh b/.github/change-versions-mac.sh index 2257d348ae..50ec3b17ed 100755 --- a/.github/change-versions-mac.sh +++ b/.github/change-versions-mac.sh @@ -16,7 +16,7 @@ sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/ sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml -sed -i '' -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 +sed -i '' -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 # sed -i '' -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml sed -i '' -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile sed -i '' -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile diff --git a/.github/change-versions.sh b/.github/change-versions.sh index 77ecbc6985..699b87b0d7 100755 --- a/.github/change-versions.sh +++ b/.github/change-versions.sh @@ -17,7 +17,7 @@ sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/pac sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml -sed -i -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 +sed -i -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 # sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile From efd9fbe1ca7b72de499fbc6405ec77f2ef8ff734 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 17:03:49 +0100 Subject: [PATCH 26/48] update svelte-kit to 2.9.0 --- frontend/package-lock.json | 64 +++++++++-------- frontend/package.json | 2 +- frontend/src/app.html | 108 +++++++++++++++-------------- frontend/src/routes/+layout.svelte | 2 + 4 files changed, 94 insertions(+), 82 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 024b5e634c..ba887fb722 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -88,7 +88,7 @@ "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", "@sveltejs/adapter-static": "^3.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/kit": "^2.9.0", "@sveltejs/package": "^2.2.2", "@sveltejs/vite-plugin-svelte": "^3.0.0", "@tailwindcss/forms": "^0.5.3", @@ -3897,10 +3897,11 @@ } }, "node_modules/@polka/url": { - "version": "1.0.0-next.24", - "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.24.tgz", - "integrity": "sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ==", - "dev": true + "version": "1.0.0-next.28", + "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.28.tgz", + "integrity": "sha512-8LduaNlMZGwdZ6qWrKlfa+2M4gahzFkprZiAt2TF8uS0qQgBizKXpXURqvTJ4WtmupWxaLqjRb2UCTe72mu+Aw==", + "dev": true, + "license": "MIT" }, "node_modules/@popperjs/core": { "version": "2.11.8", @@ -4223,23 +4224,24 @@ } }, "node_modules/@sveltejs/kit": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.5.0.tgz", - "integrity": "sha512-1uyXvzC2Lu1FZa30T4y5jUAC21R309ZMRG0TPt+PPPbNUoDpy8zSmSNVWYaBWxYDqLGQ5oPNWvjvvF2IjJ1jmA==", + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.9.0.tgz", + "integrity": "sha512-W3E7ed3ChB6kPqRs2H7tcHp+Z7oiTFC6m+lLyAQQuyXeqw6LdNuuwEUla+5VM0OGgqQD+cYD6+7Xq80vVm17Vg==", "dev": true, "hasInstallScript": true, + "license": "MIT", "dependencies": { "@types/cookie": "^0.6.0", "cookie": "^0.6.0", - "devalue": "^4.3.2", - "esm-env": "^1.0.0", - "import-meta-resolve": "^4.0.0", + "devalue": "^5.1.0", + "esm-env": "^1.2.1", + "import-meta-resolve": "^4.1.0", "kleur": "^4.1.5", "magic-string": "^0.30.5", "mrmime": "^2.0.0", "sade": "^1.8.1", "set-cookie-parser": "^2.6.0", - "sirv": "^2.0.4", + "sirv": "^3.0.0", "tiny-glob": "^0.2.9" }, "bin": { @@ -4249,9 +4251,9 @@ "node": ">=18.13" }, "peerDependencies": { - "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0", "svelte": "^4.0.0 || ^5.0.0-next.0", - "vite": "^5.0.3" + "vite": "^5.0.3 || ^6.0.0" } }, "node_modules/@sveltejs/package": { @@ -6434,10 +6436,11 @@ } }, "node_modules/devalue": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/devalue/-/devalue-4.3.2.tgz", - "integrity": "sha512-KqFl6pOgOW+Y6wJgu80rHpo2/3H07vr8ntR9rkkFIRETewbf5GaYYcakYfiKz89K+sLsuPkQIZaXDMjUObZwWg==", - "dev": true + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.1.1.tgz", + "integrity": "sha512-maua5KUiapvEwiEAe+XnlZ3Rh0GD+qI1J/nb9vrJc3muPXvcF/8gXYTWF76+5DAqHyDUtOIImEuo0YKE9mshVw==", + "dev": true, + "license": "MIT" }, "node_modules/devlop": { "version": "1.1.0", @@ -6990,9 +6993,10 @@ } }, "node_modules/esm-env": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.0.0.tgz", - "integrity": "sha512-Cf6VksWPsTuW01vU9Mk/3vRue91Zevka5SjyNf3nEpokFRuqt/KjUQoGAwq9qMmhpLTHmXzSIrFRw8zxWzmFBA==" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.1.tgz", + "integrity": "sha512-U9JedYYjCnadUlXk7e1Kr+aENQhtUaoaV9+gZm1T8LC/YBAPJx3NSPIAurFOC0U5vrdSevnUJS2/wUVxGwPhng==", + "license": "MIT" }, "node_modules/esm-env-robust": { "version": "0.0.3", @@ -7932,10 +7936,11 @@ } }, "node_modules/import-meta-resolve": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.0.0.tgz", - "integrity": "sha512-okYUR7ZQPH+efeuMJGlq4f8ubUgO50kByRPyt/Cy1Io4PSRsPjxME+YlVaCOx+NIToW7hCsZNFJyTPFFKepRSA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz", + "integrity": "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==", "dev": true, + "license": "MIT", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -9781,6 +9786,7 @@ "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.0.tgz", "integrity": "sha512-eu38+hdgojoyq63s+yTpN4XMBdt5l8HhMhc4VKLO9KM5caLIBvUm4thi7fFaxyTmCKeNnXZ5pAlBwCUnhA09uw==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" } @@ -11960,17 +11966,18 @@ "dev": true }, "node_modules/sirv": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/sirv/-/sirv-2.0.4.tgz", - "integrity": "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.0.tgz", + "integrity": "sha512-BPwJGUeDaDCHihkORDchNyyTvWFhcusy1XMmhEVTQTwGeybFbp8YEmB+njbPnth1FibULBSBVwCQni25XlCUDg==", "dev": true, + "license": "MIT", "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", "totalist": "^3.0.0" }, "engines": { - "node": ">= 10" + "node": ">=18" } }, "node_modules/slash": { @@ -13064,6 +13071,7 @@ "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz", "integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } diff --git a/frontend/package.json b/frontend/package.json index 847f5b4ee7..15ccede702 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -22,7 +22,7 @@ "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", "@sveltejs/adapter-static": "^3.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/kit": "^2.9.0", "@sveltejs/package": "^2.2.2", "@sveltejs/vite-plugin-svelte": "^3.0.0", "@tailwindcss/forms": "^0.5.3", diff --git a/frontend/src/app.html b/frontend/src/app.html index 7f93534ddb..718800105a 100644 --- a/frontend/src/app.html +++ b/frontend/src/app.html @@ -27,61 +27,63 @@
-
-
+
+
+
-
- - - - - + + + + + - - - - - -
- Loading... + /> + + + + + +
+ Loading... +
%sveltekit.body% diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelte index 63269d9375..dc715ebca9 100644 --- a/frontend/src/routes/+layout.svelte +++ b/frontend/src/routes/+layout.svelte @@ -15,6 +15,8 @@ intro: { x: 256 }, // toast intro fly animation settings theme: {} // css var overrides } + + document.getElementById('svelte-global-loader')?.remove() From 555851706a2d9834c9da6aa9a49905beec87732e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 17:17:11 +0100 Subject: [PATCH 27/48] chore update vite to 5.4.11 + adapters --- frontend/package-lock.json | 64 ++++++++++++++++++-------------------- frontend/package.json | 6 ++-- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ba887fb722..300cb2649c 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -87,10 +87,10 @@ "@hey-api/openapi-ts": "^0.43.0", "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", - "@sveltejs/adapter-static": "^3.0.0", + "@sveltejs/adapter-static": "^3.0.6", "@sveltejs/kit": "^2.9.0", "@sveltejs/package": "^2.2.2", - "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@sveltejs/vite-plugin-svelte": "^3.1.2", "@tailwindcss/forms": "^0.5.3", "@tailwindcss/typography": "^0.5.8", "@types/d3": "^7.4.0", @@ -130,7 +130,7 @@ "tailwindcss": "^3.4.1", "tslib": "^2.6.1", "typescript": "^5.1.3", - "vite": "^5", + "vite": "^5.4.11", "vite-plugin-circular-dependency": "^0.2.1", "vite-plugin-mkcert": "^1.17.5", "yootils": "^0.3.1" @@ -3584,9 +3584,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==" + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.20", @@ -4215,9 +4215,9 @@ } }, "node_modules/@sveltejs/adapter-static": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.1.tgz", - "integrity": "sha512-6lMvf7xYEJ+oGeR5L8DFJJrowkefTK6ZgA4JiMqoClMkKq0s6yvsd3FZfCFvX1fQ0tpCD7fkuRVHsnUVgsHyNg==", + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.6.tgz", + "integrity": "sha512-MGJcesnJWj7FxDcB/GbrdYD3q24Uk0PIL4QIX149ku+hlJuj//nxUbb0HxUTpjkecWfHjVveSUnUaQWnPRXlpg==", "dev": true, "peerDependencies": { "@sveltejs/kit": "^2.0.0" @@ -4279,17 +4279,17 @@ } }, "node_modules/@sveltejs/vite-plugin-svelte": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-3.0.2.tgz", - "integrity": "sha512-MpmF/cju2HqUls50WyTHQBZUV3ovV/Uk8k66AN2gwHogNAG8wnW8xtZDhzNBsFJJuvmq1qnzA5kE7YfMJNFv2Q==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-3.1.2.tgz", + "integrity": "sha512-Txsm1tJvtiYeLUVRNqxZGKR/mI+CzuIQuc2gn+YCs9rMTowpNZ2Nqt53JdL8KF9bLhAf2ruR/dr9eZCwdTriRA==", "dev": true, "dependencies": { - "@sveltejs/vite-plugin-svelte-inspector": "^2.0.0", + "@sveltejs/vite-plugin-svelte-inspector": "^2.1.0", "debug": "^4.3.4", "deepmerge": "^4.3.1", "kleur": "^4.1.5", - "magic-string": "^0.30.5", - "svelte-hmr": "^0.15.3", + "magic-string": "^0.30.10", + "svelte-hmr": "^0.16.0", "vitefu": "^0.2.5" }, "engines": { @@ -4301,9 +4301,9 @@ } }, "node_modules/@sveltejs/vite-plugin-svelte-inspector": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-2.0.0.tgz", - "integrity": "sha512-gjr9ZFg1BSlIpfZ4PRewigrvYmHWbDrq2uvvPB1AmTWKuM+dI1JXQSUu2pIrYLb/QncyiIGkFDFKTwJ0XqQZZg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-2.1.0.tgz", + "integrity": "sha512-9QX28IymvBlSCqsCll5t0kQVxipsfhFFL+L2t3nTWfXnddYwxBuAEtTtlaVQpRz9c37BhJjltSeY4AJSC03SSg==", "dev": true, "dependencies": { "debug": "^4.3.4" @@ -8613,14 +8613,11 @@ } }, "node_modules/magic-string": { - "version": "0.30.5", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.5.tgz", - "integrity": "sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==", + "version": "0.30.14", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.14.tgz", + "integrity": "sha512-5c99P1WKTed11ZC0HMJOj6CDIue6F8ySu+bJL+85q1zBEIY8IklrJ1eiKC2NDRh3Ct3FcvmJPyQHb9erXMTJNw==", "dependencies": { - "@jridgewell/sourcemap-codec": "^1.4.15" - }, - "engines": { - "node": ">=12" + "@jridgewell/sourcemap-codec": "^1.5.0" } }, "node_modules/map-obj": { @@ -12649,9 +12646,9 @@ } }, "node_modules/svelte-hmr": { - "version": "0.15.3", - "resolved": "https://registry.npmjs.org/svelte-hmr/-/svelte-hmr-0.15.3.tgz", - "integrity": "sha512-41snaPswvSf8TJUhlkoJBekRrABDXDMdpNpT2tfHIv4JuhgvHqLMhEPGtaQn0BmbNSTkuz2Ed20DF2eHw0SmBQ==", + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/svelte-hmr/-/svelte-hmr-0.16.0.tgz", + "integrity": "sha512-Gyc7cOS3VJzLlfj7wKS0ZnzDVdv3Pn2IuVeJPk9m2skfhcu5bq3wtIZyQGggr7/Iim5rH5cncyQft/kRLupcnA==", "dev": true, "engines": { "node": "^12.20 || ^14.13.1 || >= 16" @@ -13403,15 +13400,14 @@ } }, "node_modules/vite": { - "version": "5.4.0", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.0.tgz", - "integrity": "sha512-5xokfMX0PIiwCMCMb9ZJcMyh5wbBun0zUzKib+L65vAZ8GY9ePZMXxFrHbr/Kyll2+LSCY7xtERPpxkBDKngwg==", + "version": "5.4.11", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.11.tgz", + "integrity": "sha512-c7jFQRklXua0mTzneGW9QVyxFjUgwcihC4bXEtujIo2ouWCe1Ajt/amn2PCxYnhYfd5k09JX3SB7OYWFKYqj8Q==", "dev": true, - "license": "MIT", "dependencies": { "esbuild": "^0.21.3", - "postcss": "^8.4.40", - "rollup": "^4.13.0" + "postcss": "^8.4.43", + "rollup": "^4.20.0" }, "bin": { "vite": "bin/vite.js" diff --git a/frontend/package.json b/frontend/package.json index 15ccede702..3c4aa926ff 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,10 +21,10 @@ "@hey-api/openapi-ts": "^0.43.0", "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", - "@sveltejs/adapter-static": "^3.0.0", + "@sveltejs/adapter-static": "^3.0.6", "@sveltejs/kit": "^2.9.0", "@sveltejs/package": "^2.2.2", - "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@sveltejs/vite-plugin-svelte": "^3.1.2", "@tailwindcss/forms": "^0.5.3", "@tailwindcss/typography": "^0.5.8", "@types/d3": "^7.4.0", @@ -64,7 +64,7 @@ "tailwindcss": "^3.4.1", "tslib": "^2.6.1", "typescript": "^5.1.3", - "vite": "^5", + "vite": "^5.4.11", "vite-plugin-circular-dependency": "^0.2.1", "vite-plugin-mkcert": "^1.17.5", "yootils": "^0.3.1" From 667167a022ed31cb67a6c189cc5bd54b09473f4f Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Thu, 5 Dec 2024 17:42:33 +0100 Subject: [PATCH 28/48] fix: fix `flow_node` uniqueness (#4850) `jsonb` comparison wasn't working as expected, and duplicated entries were inserted within `flow_node`. To resolve this add a second hash column, `hash_v2` with a unique default for uniqueness, and use this new column to ensure unique entries. The previous hash column is left for backward compatibility. Duplicated entries already insterted will remain as is without breaking, and only new ones will preserve uniqueness. --- ...ca0e75f30faa8c39ad4754bd568ae6f210806.json | 27 ++++++++++++ ...05135747_fix_flow_node_uniqueness.down.sql | 2 + ...1205135747_fix_flow_node_uniqueness.up.sql | 4 ++ .../windmill-worker/src/worker_lockfiles.rs | 43 +++++++------------ 4 files changed, 48 insertions(+), 28 deletions(-) create mode 100644 backend/.sqlx/query-83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806.json create mode 100644 backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql create mode 100644 backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql diff --git a/backend/.sqlx/query-83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806.json b/backend/.sqlx/query-83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806.json new file mode 100644 index 0000000000..4e1242428a --- /dev/null +++ b/backend/.sqlx/query-83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bpchar", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [ + false + ] + }, + "hash": "83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806" +} diff --git a/backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql b/backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql new file mode 100644 index 0000000000..7a5d08f3f8 --- /dev/null +++ b/backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE flow_node DROP COLUMN hash_v2; diff --git a/backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql b/backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql new file mode 100644 index 0000000000..799f33d341 --- /dev/null +++ b/backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql @@ -0,0 +1,4 @@ +-- Add up migration script here +CREATE SEQUENCE IF NOT EXISTS flow_node_hash_seq; +ALTER TABLE flow_node ALTER COLUMN hash DROP NOT NULL; +ALTER TABLE flow_node ADD COLUMN hash_v2 CHAR(64) NOT NULL UNIQUE DEFAULT to_hex(nextval('flow_node_hash_seq')); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 71834a9f04..0ea7456ec3 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -4,6 +4,7 @@ use std::path::{Component, Path, PathBuf}; use async_recursion::async_recursion; use serde_json::value::RawValue; use serde_json::{json, Value}; +use sha2::Digest; use sqlx::types::Json; use uuid::Uuid; use windmill_common::error::Error; @@ -1025,46 +1026,32 @@ async fn insert_flow_node<'c>( flow: Option<&Json>>, ) -> Result<(sqlx::Transaction<'c, sqlx::Postgres>, FlowNodeId)> { let hash = { - use std::hash::{DefaultHasher, Hash, Hasher}; - - let mut hasher = DefaultHasher::new(); - code.hash(&mut hasher); - lock.hash(&mut hasher); - flow.inspect(|flow| flow.get().hash(&mut hasher)); - hasher.finish() as i64 + let mut hasher = sha2::Sha256::new(); + hasher.update(path); + hasher.update(workspace_id); + hasher.update(code.unwrap_or(&Default::default())); + hasher.update(lock.unwrap_or(&Default::default())); + hasher.update(flow.unwrap_or(&Default::default()).get()); + format!("{:x}", hasher.finalize()) }; // Insert the flow node if it doesn't exist. let id = sqlx::query_scalar!( r#" - WITH existing AS ( - SELECT id FROM flow_node - WHERE hash = $1 AND path = $2 AND workspace_id = $3 - AND (code IS NOT DISTINCT FROM $4) - AND (lock IS NOT DISTINCT FROM $5) - AND (flow IS NOT DISTINCT FROM $6) - LIMIT 1 - ), - inserted AS ( - INSERT INTO flow_node (hash, path, workspace_id, code, lock, flow) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT DO NOTHING - RETURNING id - ) - SELECT id FROM existing - UNION ALL - SELECT id FROM inserted + INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id + RETURNING id "#, - hash, path, workspace_id, - code, + hash, lock, + code, flow as Option<&Json>> ) .fetch_one(&mut *tx) - .await? - .ok_or(error::Error::InternalErr("Failed to cache".to_string()))?; + .await?; Ok((tx, FlowNodeId(id))) } From 97901182b6755e6b1f4bbbb7ee7f62b334cc7110 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 17:54:39 +0100 Subject: [PATCH 29/48] nit cache and skip on input node --- .../src/lib/components/FlowGraphViewer.svelte | 2 + .../components/FlowStatusViewerInner.svelte | 2 + frontend/src/lib/components/Section.svelte | 4 +- frontend/src/lib/components/Toggle.svelte | 4 +- .../flows/content/FlowSettings.svelte | 104 ++++++++++-------- .../flows/map/FlowModuleSchemaMap.svelte | 2 + .../components/flows/map/VirtualItem.svelte | 30 ++++- .../lib/components/graph/FlowGraphV2.svelte | 6 +- .../graph/renderers/nodes/InputNode.svelte | 4 + .../[job]/[resume]/[hmac]/+page.svelte | 2 + 10 files changed, 104 insertions(+), 56 deletions(-) diff --git a/frontend/src/lib/components/FlowGraphViewer.svelte b/frontend/src/lib/components/FlowGraphViewer.svelte index 326e1275b6..f69c1c9450 100644 --- a/frontend/src/lib/components/FlowGraphViewer.svelte +++ b/frontend/src/lib/components/FlowGraphViewer.svelte @@ -36,6 +36,8 @@ > - {#if collapsable && collapsed} - - {/if} +
diff --git a/frontend/src/lib/components/Toggle.svelte b/frontend/src/lib/components/Toggle.svelte index 73638bce9c..068082d2f7 100644 --- a/frontend/src/lib/components/Toggle.svelte +++ b/frontend/src/lib/components/Toggle.svelte @@ -104,9 +104,7 @@ {#if eeOnly && disabled} - + EE only Enterprise Edition only feature diff --git a/frontend/src/lib/components/flows/content/FlowSettings.svelte b/frontend/src/lib/components/flows/content/FlowSettings.svelte index 51a2410b31..403ac55f1b 100644 --- a/frontend/src/lib/components/flows/content/FlowSettings.svelte +++ b/frontend/src/lib/components/flows/content/FlowSettings.svelte @@ -138,7 +138,7 @@ right: 'Worker group tag (queue)', rightTooltip: "When a worker group tag is defined at the flow level, any steps inside the flow will run on any worker group that listen to that tag, regardless of the steps tag. If no worker group tags is defined, the flow controls will be executed with the default tag 'flow' and the steps will be executed with their respective tag", - rightDocumentationLink: 'https://www.windmill.dev/docs/core_concepts/worker_groups' + rightDocumentationLink: 'https://www.windmill.dev/docs/core_concepts/worker_groups' }} class="py-1" /> @@ -185,7 +185,8 @@ }} options={{ right: 'Cache the results for each possible inputs', - rightTooltip: 'When enabled, the flow will cache the results of the flow for each possible set of inputs.', + rightTooltip: + 'When enabled, the flow will cache the results of the flow for each possible set of inputs.', rightDocumentationLink: 'https://www.windmill.dev/docs/flows/cache#cache-flows' }} class="py-1" @@ -226,7 +227,8 @@ rightTooltip: 'If the inputs meet the predefined condition, the flow will not run.' + 'to decide if the flow should stop early.', - rightDocumentationLink: 'https://www.windmill.dev/docs/flows/early_stop#early-stop-for-flow' + rightDocumentationLink: + 'https://www.windmill.dev/docs/flows/early_stop#early-stop-for-flow' }} class="py-1" /> @@ -316,7 +318,8 @@ 'Steps will share a folder at `./shared` in which they can store heavier data and ' + 'pass them to the next step. Beware that the `./shared` folder is not ' + 'preserved across suspends and sleeps.', - rightDocumentationLink: 'https://www.windmill.dev/docs/core_concepts/persistent_storage/within_windmill#shared-directory' + rightDocumentationLink: + 'https://www.windmill.dev/docs/core_concepts/persistent_storage/within_windmill#shared-directory' }} class="py-1" /> @@ -339,7 +342,8 @@ right: 'Make runs invisible to others', rightTooltip: 'When this option is enabled, manual executions of this script are invisible to users other than the user running it, including the owner(s). This setting can be overridden when this script is run manually from the advanced menu.', - rightDocumentationLink: 'https://www.windmill.dev/docs/core_concepts/monitor_past_and_future_runs#invisible-runs' + rightDocumentationLink: + 'https://www.windmill.dev/docs/core_concepts/monitor_past_and_future_runs#invisible-runs' }} class="py-1" /> @@ -365,27 +369,29 @@ {#if customUi?.settingsTabs?.concurrency != false}
- { - if ($flowStore.value.concurrent_limit) { - $flowStore.value.concurrent_limit = undefined - } else { - $flowStore.value.concurrent_limit = 1 - } - }} - options={{ - right: 'Concurrency limits', - rightTooltip: 'Allowed concurrency within a given timeframe', - rightDocumentationLink: 'https://www.windmill.dev/docs/flows/concurrency_limit' - }} - class="py-1" - eeOnly={true} - /> +
+ { + if ($flowStore.value.concurrent_limit) { + $flowStore.value.concurrent_limit = undefined + } else { + $flowStore.value.concurrent_limit = 1 + } + }} + options={{ + right: 'Concurrency limits', + rightTooltip: 'Allowed concurrency within a given timeframe', + rightDocumentationLink: 'https://www.windmill.dev/docs/flows/concurrency_limit' + }} + class="py-1" + eeOnly={true} + /> +
{#if $flowStore.value.concurrent_limit}
@@ -483,28 +489,30 @@
- { - if ($flowStore.dedicated_worker) { - $flowStore.dedicated_worker = undefined - } else { - $flowStore.dedicated_worker = true - } - }} - options={{ - right: 'Flow is run on dedicated workers', - rightTooltip: - 'When enabled, the flow will be executed on a dedicated worker.', - rightDocumentationLink: 'https://www.windmill.dev/docs/core_concepts/jobs#high-priority-jobs' - }} - class="py-1" - eeOnly={true} - /> +
+ { + if ($flowStore.dedicated_worker) { + $flowStore.dedicated_worker = undefined + } else { + $flowStore.dedicated_worker = true + } + }} + options={{ + right: 'Flow is run on dedicated workers', + rightTooltip: 'When enabled, the flow will be executed on a dedicated worker.', + rightDocumentationLink: + 'https://www.windmill.dev/docs/core_concepts/jobs#high-priority-jobs' + }} + class="py-1" + eeOnly={true} + /> +
{#if $flowStore.dedicated_worker}
diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte index 8e53a1030c..9e89f92f73 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte @@ -330,6 +330,8 @@
('FlowCopilotContext') || {} @@ -62,7 +67,30 @@
{/if}
- +
+ {#if cache} + +
+ +
+ Cached + + {/if} + {#if earlyStop} + +
+ +
+ Early stop if condition met + + {/if} +
{#if inputJson && $flowPropPickerConfig && (Object.keys(inputJson).length > 0 || alwaysPluggable)}
diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index 2e46fb309c..48f7fc9430 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -57,6 +57,8 @@ export let newFlow: boolean = false export let insertable = false + export let earlyStop: boolean = false + export let cache: boolean = false export let scroll = false export let moving: string | undefined = undefined @@ -238,7 +240,9 @@ flowModuleStates, selectedId: $selectedId, path, - newFlow + newFlow, + cache, + earlyStop }, failureModule, preprocessorModule, diff --git a/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte index 30eb352800..bdfcdd30cb 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte @@ -18,6 +18,8 @@ index: number disableAi: boolean disableMoveIds: string[] + cache: boolean + earlyStop: boolean } const { selectedId } = getContext<{ @@ -83,5 +85,7 @@ inputJson={filteredInput} prefix="flow_input" alwaysPluggable + cache={data.cache} + earlyStop={data.earlyStop} /> diff --git a/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte b/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte index eedeca636c..ca18edfe78 100644 --- a/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte +++ b/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte @@ -286,6 +286,8 @@ Date: Thu, 5 Dec 2024 19:19:58 +0100 Subject: [PATCH 30/48] Update changelog nov (#4859) * Update changelog nov * Updated link tantivy * No duplicate --- .../src/lib/components/sidebar/changelogs.ts | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/frontend/src/lib/components/sidebar/changelogs.ts b/frontend/src/lib/components/sidebar/changelogs.ts index 0c6be5542e..478c2dc64b 100644 --- a/frontend/src/lib/components/sidebar/changelogs.ts +++ b/frontend/src/lib/components/sidebar/changelogs.ts @@ -5,6 +5,71 @@ export type Changelog = { } const changelogs: Changelog[] = [ + { + label: 'Full text search on jobs and logs', + href: 'https://www.windmill.dev/changelog/instant-full-text-search-on-jobs-and-logs', + date: '2024-12-05' + }, + { + label: 'Force dark/light theme in apps', + href: 'https://www.windmill.dev/changelog/force-dark-light-theme', + date: '2024-11-28' + }, + { + label: 'Kafka triggers', + href: 'https://www.windmill.dev/changelog/kafka-triggers', + date: '2024-11-18' + }, + { + label: 'Critical channels in UI', + href: 'https://www.windmill.dev/changelog/critical-channels-ui', + date: '2024-11-15' + }, + { + label: 'Support for Mistral and Anthropic AI models', + href: 'https://www.windmill.dev/changelog/mistral-anthropic-support', + date: '2024-11-14' + }, + { + label: 'Websocket triggers', + href: 'https://www.windmill.dev/changelog/websocket-triggers', + date: '2024-11-06' + }, + { + label: 'Autoscaling', + href: 'https://www.windmill.dev/changelog/autoscaling', + date: '2024-10-28' + }, + { + label: 'File download helper', + href: 'https://www.windmill.dev/changelog/file-download-helper', + date: '2024-10-12' + }, + { + label: 'Queue metric alerts', + href: 'https://www.windmill.dev/changelog/queue-metric-alerts', + date: '2024-10-10' + }, + { + label: 'Deno 2.0', + href: 'https://www.windmill.dev/changelog/deno-2.0', + date: '2024-10-10' + }, + { + label: 'Move components inside containers with ctrl+click', + href: 'https://www.windmill.dev/changelog/move-components-inside-containers-with-ctrl', + date: '2024-10-09' + }, + { + label: 'Support workers to run natively on Windows', + href: 'https://www.windmill.dev/changelog/workers-run-natively-windows', + date: '2024-10-03' + }, + { + label: 'Quick access menu for faster component insertion', + href: 'https://www.windmill.dev/changelog/flow-quick-access-menu', + date: '2024-10-03' + }, { label: 'Custom HTTP routes', href: 'https://www.windmill.dev/changelog/http-routing', From be624b1a9a39131ede5cf9a0b1676759140fa27a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 22:45:45 +0100 Subject: [PATCH 31/48] load all branchall even if branches > 20 --- frontend/src/lib/components/FlowStatusViewerInner.svelte | 8 ++++---- frontend/src/routes/view_graph/+page.svelte | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 53fa3b0af6..9f816d42c7 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -52,6 +52,7 @@ flowJobs: string[] flowJobsSuccess: (boolean | undefined)[] length: number + branchall?: boolean } | undefined = undefined @@ -466,9 +467,7 @@ let modId = flowJobIds?.moduleId ?? '' let common = { - iteration_from: - // $localDurationStatuses?.[modId]?.iteration_from ?? - Math.max(flowJobIds.flowJobs.length - 20, 0), + iteration_from: flowJobIds?.branchall ? 0 : Math.max(flowJobIds.flowJobs.length - 20, 0), iteration_total: $localDurationStatuses?.[modId]?.iteration_total ?? flowJobIds?.length } $localDurationStatuses[modId] = { @@ -1141,7 +1140,8 @@ moduleId: mod.id, flowJobs: mod.flow_jobs, flowJobsSuccess: mod.flow_jobs_success, - length: mod.iterator?.itered?.length ?? mod.flow_jobs.length + length: mod.iterator?.itered?.length ?? mod.flow_jobs.length, + branchall: job?.raw_flow?.modules?.[i]?.value?.type == 'branchall' } : undefined} on:jobsLoaded={(e) => { diff --git a/frontend/src/routes/view_graph/+page.svelte b/frontend/src/routes/view_graph/+page.svelte index 18b49846a8..a1804647ac 100644 --- a/frontend/src/routes/view_graph/+page.svelte +++ b/frontend/src/routes/view_graph/+page.svelte @@ -27,3 +27,4 @@ ) )}>Download +the 55 From 9ecc94a268d8540224a4111ced36e235407359b2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Dec 2024 23:52:34 +0100 Subject: [PATCH 32/48] optimize for loop rendering with more than 500 iterations --- frontend/src/lib/components/FlowStatusViewerInner.svelte | 9 ++++++++- .../components/flows/content/FlowModuleEarlyStop.svelte | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 9f816d42c7..35803d53e3 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -817,6 +817,8 @@ return rec(ids, undefined) } + + let subflowsSize = 500 {#if notAnonynmous} @@ -946,7 +948,12 @@
{#each flowJobIds?.flowJobs ?? [] as loopJobId, j (loopJobId)} - {#if render} + {#if render && j + subflowsSize + 1 == (flowJobIds?.flowJobs.length ?? 0)} + + {/if} + {#if render && j + subflowsSize + 1 > (flowJobIds?.flowJobs.length ?? 0)} +
Delay
{/if} {:else if delayType === 'exponential'} {#if flowModuleRetry?.exponential}
Attempts
- +
+ + +
Multiplier
delay = multiplier * base ^ (number of attempt) @@ -127,9 +149,9 @@ multiplier, random_factor } = flowModuleRetry?.exponential || {}} - {@const cArray = Array.from({ length: cAttempts || 0 }, () => cSeconds)} + {@const cArray = Array.from({ length: Math.min(cAttempts || 0, 100) }, () => cSeconds)} {@const eArray = Array.from( - { length: eAttempts || 0 }, + { length: Math.min(eAttempts || 0, 100) }, (_, i) => (multiplier || 0) * (eSeconds || 0) ** (i + cArray.length + 1) )} {@const array = [...cArray, ...eArray]} @@ -146,7 +168,7 @@ seconds){/if} - {#each array.slice(1) as delay, i} + {#each array.slice(1, 100) as delay, i} {@const index = i + 2} {index}: @@ -163,6 +185,12 @@ {/each} + {#if (cAttempts ?? 0) > 100 || (eAttempts ?? 0) > 100} + + ... + ... + + {/if} {:else}
No retries
From 45d4fc2de71540a7a9defec9877a5b32fe63d7a6 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 6 Dec 2024 01:24:36 +0100 Subject: [PATCH 34/48] bump deno to 2.1.2 and bun to 1.1.38 --- Dockerfile | 4 ++-- docker/DockerfileSlim | 2 +- docker/DockerfileSlimEe | 2 +- lsp/Dockerfile | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index cff245d3e4..ef2f7a721e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -175,9 +175,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools COPY --from=builder /frontend/build /static_frontend COPY --from=builder /windmill/target/release/windmill ${APP}/windmill -COPY --from=denoland/deno:2.0.4 --chmod=755 /usr/bin/deno /usr/bin/deno +COPY --from=denoland/deno:2.1.2 --chmod=755 /usr/bin/deno /usr/bin/deno -COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index a1b5daca43..35b8b8ba6a 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 94cc160cae..44dd185d25 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/lsp/Dockerfile b/lsp/Dockerfile index 2d56305116..e805d48ec7 100644 --- a/lsp/Dockerfile +++ b/lsp/Dockerfile @@ -26,7 +26,7 @@ ENV GOBIN=/usr/local/go/bin RUN /usr/local/go/bin/go install -v golang.org/x/tools/gopls@latest RUN pip3 install tornado python-lsp-jsonrpc ruff-lsp -COPY --from=denoland/deno:2.0.2 --chmod=755 /usr/bin/deno /usr/bin/deno +COPY --from=denoland/deno:2.1.2 --chmod=755 /usr/bin/deno /usr/bin/deno COPY Pipfile . From f175158b9ffcc0d9c35938e61dfe3ce56d3b64c2 Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:40:32 +0100 Subject: [PATCH 35/48] fix: handle flow & workspace renames for `flow_node` (#4861) --- ...5893edb73694601544fea7cf20f45dd25a88d.json | 27 +++++++++++++++++++ ...20241206075559_flow_node_unique_2.down.sql | 3 +++ .../20241206075559_flow_node_unique_2.up.sql | 3 +++ .../windmill-worker/src/worker_lockfiles.rs | 4 +-- 4 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json create mode 100644 backend/migrations/20241206075559_flow_node_unique_2.down.sql create mode 100644 backend/migrations/20241206075559_flow_node_unique_2.up.sql diff --git a/backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json b/backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json new file mode 100644 index 0000000000..f4021d8985 --- /dev/null +++ b/backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bpchar", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d" +} diff --git a/backend/migrations/20241206075559_flow_node_unique_2.down.sql b/backend/migrations/20241206075559_flow_node_unique_2.down.sql new file mode 100644 index 0000000000..b6e6db03e6 --- /dev/null +++ b/backend/migrations/20241206075559_flow_node_unique_2.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_unique_2; +ALTER TABLE flow_node ADD CONSTRAINT flow_node_hash_v2_key UNIQUE (hash_v2); diff --git a/backend/migrations/20241206075559_flow_node_unique_2.up.sql b/backend/migrations/20241206075559_flow_node_unique_2.up.sql new file mode 100644 index 0000000000..4e8383bd89 --- /dev/null +++ b/backend/migrations/20241206075559_flow_node_unique_2.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +ALTER TABLE flow_node ADD CONSTRAINT flow_node_unique_2 UNIQUE (path, workspace_id, hash_v2); +ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_hash_v2_key; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 0ea7456ec3..ec739a88a2 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1027,8 +1027,6 @@ async fn insert_flow_node<'c>( ) -> Result<(sqlx::Transaction<'c, sqlx::Postgres>, FlowNodeId)> { let hash = { let mut hasher = sha2::Sha256::new(); - hasher.update(path); - hasher.update(workspace_id); hasher.update(code.unwrap_or(&Default::default())); hasher.update(lock.unwrap_or(&Default::default())); hasher.update(flow.unwrap_or(&Default::default()).get()); @@ -1040,7 +1038,7 @@ async fn insert_flow_node<'c>( r#" INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow) VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id + ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id RETURNING id "#, path, From 3c4408e3dbf8a37c0b862805b1991af3d39ac054 Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:26:36 +0100 Subject: [PATCH 36/48] feat(cache): refurbish fs backed cache (#4863) * feat(cache): refurbish fs backed cache * add `cached` exemple --- backend/windmill-common/src/cache.rs | 265 +++++++++++++++++++------ backend/windmill-common/src/flows.rs | 6 + backend/windmill-common/src/scripts.rs | 6 + 3 files changed, 213 insertions(+), 64 deletions(-) diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 9810e83be5..9a81c7dace 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -1,25 +1,149 @@ use crate::error; +use std::future::Future; +use std::hash::Hash; use std::path::{Path, PathBuf}; -use quick_cache::sync::Cache; +use quick_cache::Equivalent; +use serde::{Deserialize, Serialize}; use sqlx::PgExecutor; +pub use const_format::concatcp; +pub use lazy_static::lazy_static; +pub use quick_cache::sync::Cache; + /// Cache directory for windmill server/worker(s). pub const CACHE_DIR: &str = "/tmp/windmill/cache/"; +/// A file-system backed concurrent cache. +pub struct FsBackedCache { + cache: Cache, + root: &'static str, +} + +impl FsBackedCache { + /// Create a new file-system backed cache with `items_capacity` capacity. + /// The cache will be stored in the `root` directory. + pub fn new(root: &'static str, items_capacity: usize) -> Self { + Self { cache: Cache::new(items_capacity), root } + } + + /// Gets or inserts an item in the cache with key `key`. + pub async fn get_or_insert_async<'a, Q, F>(&'a self, key: &Q, with: F) -> error::Result + where + Q: Hash + Equivalent + ToOwned + Copy + Into, + F: Future>, + { + self.cache + .get_or_insert_async( + key, + fs::import_or_insert_with(self.root, (*key).into(), with), + ) + .await + } +} + +/// Like [`lazy_static`]`, but for file-system backed caches. +/// +/// # Example +/// ```rust +/// use windmill_common::make_static; +/// +/// make_static! { +/// /// String cache with a maximum capacity of 1000 items stored in the +/// /// "subdirectory" directory. +/// static ref CACHE: { u64 => String } in "subdirectory" <= 1000; +/// /// Another cache. +/// static ref ANOTHER_CACHE: { u64 => Vec } in "another" <= 100; +/// } +/// ``` +#[macro_export] +macro_rules! make_static { + { $( $(#[$attr:meta])* static ref $name:ident: { $Key:ty => $Val:ty } in $root:literal <= $cap:literal; )+ } => { + $crate::cache::lazy_static! { + $( + $(#[$attr])* + static ref $name: $crate::cache::FsBackedCache<$Key, $Val> = + $crate::cache::FsBackedCache::new( + $crate::cache::concatcp!($crate::cache::CACHE_DIR, $root), + $cap + ); + )+ + } + }; +} + +// re-export: +pub use make_static; + +/// Create an anonymous file-system backed cache for one-time use. +/// +/// # Example +/// ```rust +/// use windmill_common::anon; +/// let cache = anon!({ u64 => String } in "subdirectory" <= 1000); +/// ``` +#[macro_export] +macro_rules! anon { + ({ $Key:ty => $Val:ty } in $root:literal <= $cap:literal) => {{ + $crate::cache::make_static! { + static ref __ANON__: { $Key => $Val } in $root <= $cap; + } + + &__ANON__ + }}; +} + +// re-export: +pub use anon; + +pub mod future { + use super::*; + + /// Extension trait for futures that can be cached. + pub trait FutureCachedExt: + Future> + Sized + { + /// Get or insert the future result in the cache. + /// + /// # Example + /// ```rust + /// use windmill_common::cache::{self, future::FutureCachedExt}; + /// + /// async { + /// let result = std::future::ready(Ok(42)) + /// .cached(cache::anon!({ u64 => u64 } in "test" <= 1), &42) + /// .await; + /// + /// assert_eq!(result.unwrap(), 42); + /// }; + /// ``` + fn cached( + self, + cache: &FsBackedCache, + key: &Q, + ) -> impl Future> + where + Q: Hash + Equivalent + ToOwned + Copy + Into, + { + cache.get_or_insert_async(key, self) + } + } + + impl> + Sized> + FutureCachedExt for F + { + } +} + pub mod flow { use super::*; use crate::flows::{FlowNodeId, FlowValue}; - /// Cache directory for windmill server/worker(s) flow nodes. - pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "flow"); - - lazy_static::lazy_static! { + make_static! { /// Flow node cache. - /// FIXME: This should be a static but [`Cache`] does not have a const constructor. /// FIXME: Use `Arc` for cheap cloning. - static ref CACHE: Cache = Cache::new(1000); + static ref CACHE: { FlowNodeId => Val } in "flow" <= 1000; } /// Flow node cache value. @@ -75,38 +199,35 @@ pub mod flow { // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. CACHE - .get_or_insert_async( - &node, - fs::import_or_insert_with(CACHE_DIR, node.0 as u64, async { - sqlx::query!( - "SELECT \ + .get_or_insert_async(&node, async { + sqlx::query!( + "SELECT \ lock AS \"lock: String\", \ code AS \"code: String\", \ flow::text AS \"flow: Box\" \ FROM flow_node WHERE id = $1 LIMIT 1", - node.0, - ) - .fetch_one(e) - .await - .map_err(Into::into) - .and_then(|r| { - Ok(Val { - lock: r - .lock - .and_then(|x| if x.is_empty() { None } else { Some(x) }), - code: r.code, - flow: match r.flow { - None => None, - Some(flow) => serde_json::from_str(&flow).map_err(|err| { - error::Error::InternalErr(format!( - "Unable to parse flow value: {err:?}" - )) - })?, - }, - }) + node.0, + ) + .fetch_one(e) + .await + .map_err(Into::into) + .and_then(|r| { + Ok(Val { + lock: r + .lock + .and_then(|x| if x.is_empty() { None } else { Some(x) }), + code: r.code, + flow: match r.flow { + None => None, + Some(flow) => serde_json::from_str(&flow).map_err(|err| { + error::Error::InternalErr(format!( + "Unable to parse flow value: {err:?}" + )) + })?, + }, }) - }), - ) + }) + }) .await } @@ -164,14 +285,10 @@ pub mod script { use super::*; use crate::scripts::{ScriptHash, ScriptLang}; - /// Cache directory for windmill server/worker(s) scripts. - pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "script"); - - lazy_static::lazy_static! { + make_static! { /// Scripts cache. - /// FIXME: This should be a static but [`Cache`] does not have a const constructor. /// FIXME: Use `Arc` for cheap cloning. - static ref CACHE: Cache = Cache::new(1000); + static ref CACHE: { ScriptHash => Val } in "script" <= 1000; } /// Script cache value. @@ -197,34 +314,31 @@ pub mod script { // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. CACHE - .get_or_insert_async( - &hash, - fs::import_or_insert_with(CACHE_DIR, hash.0 as u64, async { - sqlx::query!( - "SELECT \ + .get_or_insert_async(&hash, async { + sqlx::query!( + "SELECT \ lock AS \"lock: String\", \ content AS \"code!: String\", language AS \"language: Option\", \ envs AS \"envs: Vec\", \ codebase AS \"codebase: String\" \ FROM script WHERE hash = $1 AND workspace_id = $2 LIMIT 1", - hash.0, - workspace_id, - ) - .fetch_one(e) - .await - .map_err(Into::into) - .map(|r| Val { - lock: r - .lock - .and_then(|x| if x.is_empty() { None } else { Some(x) }), - code: r.code, - language: r.language, - envs: r.envs, - codebase: r.codebase, - }) - }), - ) + hash.0, + workspace_id, + ) + .fetch_one(e) + .await + .map_err(Into::into) + .map(|r| Val { + lock: r + .lock + .and_then(|x| if x.is_empty() { None } else { Some(x) }), + code: r.code, + language: r.language, + envs: r.envs, + codebase: r.codebase, + }) + }) .await } @@ -283,8 +397,6 @@ pub mod script { mod fs { use super::*; - use std::future::Future; - use std::fs::{self, OpenOptions}; use std::io::{Read, Write}; @@ -363,4 +475,29 @@ mod fs { } Ok(data) } + + // Auto-implement `Bundle` for all `serde` serializable types. + + impl Item for () { + fn path(&self, root: &Path) -> PathBuf { + root.join("self.json") + } + } + + impl Deserialize<'de> + Serialize + Default> Bundle for T { + type Item = (); + + fn items() -> &'static [Self::Item] { + &[()] + } + + fn import(&mut self, _: Self::Item, data: Vec) -> error::Result<()> { + *self = serde_json::from_slice(&data)?; + Ok(()) + } + + fn export(&self, _: Self::Item) -> error::Result>> { + Ok(Some(serde_json::to_vec(self)?)) + } + } } diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index cd67f741a2..29edd4db30 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -407,6 +407,12 @@ pub enum InputTransform { #[serde(transparent)] pub struct FlowNodeId(pub i64); +impl Into for FlowNodeId { + fn into(self) -> u64 { + self.0 as u64 + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Branch { #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 1a11b5a701..9d51bbb830 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -75,6 +75,12 @@ impl ScriptLang { #[sqlx(transparent)] pub struct ScriptHash(pub i64); +impl Into for ScriptHash { + fn into(self) -> u64 { + self.0 as u64 + } +} + #[derive(PartialEq, sqlx::Type)] #[sqlx(transparent, no_pg_array)] pub struct ScriptHashes(pub Vec); From 2bc4934c4f38ca2631eb99140d89a0b3d3b3a2cf Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:37:45 +0100 Subject: [PATCH 37/48] feat: add db storage for app inline scripts (#4837) --- ...9177a87e1accd192402e21db5ae09c3498ab0.json | 3 +- ...54799f5e8122adaf35465cb16c3dc795bdc3b.json | 26 ++ ...242cf3a6287e7b4548c7f01ad888230c27013.json | 22 ++ ...98a03f751b246c40daf056fced0fd91f6dd73.json | 3 +- ...ffe0ba043565790865c6a6f52fab6ce340d2c.json | 23 ++ ...31899161e80c239d56eb657d73bbf4272939b.json | 28 ++ ...914fca6fc0a8787e4f8d520766b1c89f23602.json | 15 + ...b6b493e53583017c18e2ab44f44125c52d548.json | 3 +- ...66b142fa628b8983b966357deb3d5e0a1df3c.json | 28 ++ .../20241202095902_app_script.down.sql | 3 + .../20241202095902_app_script.up.sql | 26 ++ backend/windmill-api/openapi.yaml | 21 ++ backend/windmill-api/src/apps.rs | 318 ++++++++++-------- backend/windmill-common/src/apps.rs | 13 +- backend/windmill-common/src/cache.rs | 90 +++++ backend/windmill-common/src/jobs.rs | 8 + backend/windmill-queue/src/jobs.rs | 23 +- backend/windmill-worker/src/worker.rs | 15 + .../windmill-worker/src/worker_lockfiles.rs | 101 +++++- .../helpers/RunnableComponent.svelte | 11 +- frontend/src/lib/components/apps/types.ts | 2 + .../(logged)/apps/get/[...path]/+page.svelte | 2 +- 22 files changed, 632 insertions(+), 152 deletions(-) create mode 100644 backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json create mode 100644 backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json create mode 100644 backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json create mode 100644 backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json create mode 100644 backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json create mode 100644 backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json create mode 100644 backend/migrations/20241202095902_app_script.down.sql create mode 100644 backend/migrations/20241202095902_app_script.up.sql diff --git a/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json b/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json index 8fdca62eca..1b1c6b504b 100644 --- a/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json +++ b/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json @@ -44,7 +44,8 @@ "deploymentcallback", "singlescriptflow", "flowscript", - "flownode" + "flownode", + "appscript" ] } } diff --git a/backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json b/backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json new file mode 100644 index 0000000000..efe58b7a16 --- /dev/null +++ b/backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO app_script (app, hash, lock, code, code_sha256)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Bpchar", + "Text", + "Text", + "Bpchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b" +} diff --git a/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json b/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json new file mode 100644 index 0000000000..5c43791411 --- /dev/null +++ b/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT format('rawscript/%s', code_sha256) as \"path!: String\"\n FROM app_script WHERE id = $1 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!: String", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013" +} diff --git a/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json b/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json index 20d76cbed4..2f80c0c818 100644 --- a/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json +++ b/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json @@ -34,7 +34,8 @@ "deploymentcallback", "singlescriptflow", "flowscript", - "flownode" + "flownode", + "appscript" ] } } diff --git a/backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json b/backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json new file mode 100644 index 0000000000..c9cd8219ee --- /dev/null +++ b/backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT policy as \"policy: sqlx::types::Json>\"\n FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "policy: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c" +} diff --git a/backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json b/backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json new file mode 100644 index 0000000000..450683521d --- /dev/null +++ b/backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT app_id, value FROM app_version WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "value", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b" +} diff --git a/backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json b/backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json new file mode 100644 index 0000000000..0d76f9a721 --- /dev/null +++ b/backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app_version_lite (id, value) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602" +} diff --git a/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json b/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json index 4092b60b71..e48e3b041d 100644 --- a/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json +++ b/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json @@ -48,7 +48,8 @@ "deploymentcallback", "singlescriptflow", "flowscript", - "flownode" + "flownode", + "appscript" ] } } diff --git a/backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json b/backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json new file mode 100644 index 0000000000..49fe5cf6b8 --- /dev/null +++ b/backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "lock", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "code", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c" +} diff --git a/backend/migrations/20241202095902_app_script.down.sql b/backend/migrations/20241202095902_app_script.down.sql new file mode 100644 index 0000000000..c493d82176 --- /dev/null +++ b/backend/migrations/20241202095902_app_script.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP TABLE IF EXISTS app_version_lite; +DROP TABLE IF EXISTS app_script; diff --git a/backend/migrations/20241202095902_app_script.up.sql b/backend/migrations/20241202095902_app_script.up.sql new file mode 100644 index 0000000000..5c3e6a2e0b --- /dev/null +++ b/backend/migrations/20241202095902_app_script.up.sql @@ -0,0 +1,26 @@ +-- Add up migration script here +ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'appscript'; + +-- Same as `app_version` but with a "lite" value (w/ `inlineScript.{code,lock}`). +CREATE TABLE app_version_lite ( + id BIGSERIAL PRIMARY KEY, + value JSONB, + FOREIGN KEY (id) REFERENCES app_version (id) ON DELETE CASCADE +); + +GRANT ALL ON app_version_lite TO windmill_user; +GRANT ALL ON app_version_lite TO windmill_admin; + +-- App `inlineScript`. +CREATE TABLE app_script ( + id BIGSERIAL PRIMARY KEY, + app BIGSERIAL NOT NULL, + hash CHAR(64) NOT NULL UNIQUE, -- sha256 of `app`, `lock`, `code`. + lock TEXT, + code TEXT NOT NULL, + code_sha256 CHAR(64) NOT NULL, -- used to retrieve the policy. + FOREIGN KEY (app) REFERENCES app (id) ON DELETE CASCADE +); + +GRANT ALL ON app_script TO windmill_user; +GRANT ALL ON app_script TO windmill_admin; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 95c06d5aae..ccab668051 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -5467,6 +5467,23 @@ paths: schema: $ref: "#/components/schemas/AppWithLastVersion" + /w/{workspace}/apps/get/lite/{path}: + get: + summary: get app lite by path + operationId: getAppLiteByPath + tags: + - app + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: app lite details + content: + application/json: + schema: + $ref: "#/components/schemas/AppWithLastVersion" + /w/{workspace}/apps/get/draft/{path}: get: summary: get app by path with draft @@ -5791,6 +5808,8 @@ paths: #flow: flow/ path: type: string + version: + type: integer args: {} raw_code: type: object @@ -5808,6 +5827,8 @@ paths: required: - content - language + id: + type: integer force_viewer_static_fields: type: object force_viewer_one_of_fields: diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index e1a9421252..af167d9a4a 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; /* * Author: Ruben Fiszel @@ -31,6 +31,7 @@ use axum::{ routing::{delete, get, post}, Router, }; +use futures::future::{FutureExt, TryFutureExt}; use hyper::StatusCode; #[cfg(feature = "parquet")] use itertools::Itertools; @@ -50,7 +51,8 @@ use windmill_audit::ActionKind; #[cfg(feature = "parquet")] use windmill_common::s3_helpers::build_object_store_client; use windmill_common::{ - apps::ListAppQuery, + apps::{AppScriptId, ListAppQuery}, + cache::{self, future::FutureCachedExt}, db::UserDB, error::{to_anyhow, Error, JsonResult, Result}, jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode}, @@ -71,6 +73,7 @@ pub fn workspaced_service() -> Router { .route("/list", get(list_apps)) .route("/list_search", get(list_search_apps)) .route("/get/p/*path", get(get_app)) + .route("/get/lite/*path", get(get_app_lite)) .route("/get/draft/*path", get(get_app_w_draft)) .route("/secret_of/*path", get(get_secret_id)) .route("/get/v/*id", get(get_app_by_id)) @@ -191,15 +194,16 @@ pub type StaticFields = HashMap>; pub type OneOfFields = HashMap>>; pub type AllowUserResources = Vec; -#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] #[serde(rename_all = "lowercase")] pub enum ExecutionMode { + #[default] Anonymous, Publisher, Viewer, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct PolicyTriggerableInputs { static_inputs: StaticFields, one_of_inputs: OneOfFields, @@ -215,7 +219,7 @@ pub struct S3Input { file_key_regex: String, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct Policy { pub on_behalf_of: Option, pub on_behalf_of_email: Option, @@ -410,6 +414,33 @@ async fn get_app( Ok(Json(app)) } +async fn get_app_lite( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + + let app_o = sqlx::query_as::<_, AppWithLastVersion>( + "SELECT app.id, app.path, app.summary, app.versions, app.policy, + app.extra_perms, coalesce(app_version_lite.value::json, app_version.value) as value, + app_version.created_at, app_version.created_by, NULL as starred + FROM app, app_version + LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id + WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]", + ) + .bind(path.to_owned()) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + + tx.commit().await?; + + let app = not_found_if_none(app_o, "App", path)?; + Ok(Json(app)) +} + async fn get_app_w_draft( authed: ApiAuthed, Extension(user_db): Extension, @@ -583,8 +614,9 @@ async fn get_public_app_by_secret( let app_o = sqlx::query_as::<_, AppWithLastVersion>( "SELECT app.id, app.path, app.summary, app.versions, app.policy, - null as extra_perms, app_version.value, + null as extra_perms, coalesce(app_version_lite.value::json, app_version.value::json) as value, app_version.created_at, app_version.created_by from app, app_version + LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]") .bind(&id) .bind(&w_id) @@ -1144,6 +1176,10 @@ async fn update_app( #[derive(Debug, Deserialize, Clone)] pub struct ExecuteApp { + /// The app version to execute. Fallback to `path` if not provided. + pub version: Option, + /// The app script id (from the `app_script` table) to execute. + pub id: Option, pub args: HashMap>, // - script: script/ // - flow: flow/ @@ -1206,6 +1242,22 @@ async fn get_on_behalf_details_from_policy_and_authed( Ok((username, permissioned_as, email)) } +/// Convert the triggerables from the old format to the new format. +fn empty_triggerables(mut policy: Policy) -> Policy { + use std::mem::take; + if let Some(triggerables) = take(&mut policy.triggerables) { + let mut triggerables_v2 = take(&mut policy.triggerables_v2).unwrap_or_default(); + for (k, static_inputs) in triggerables.into_iter() { + triggerables_v2.insert( + k, + PolicyTriggerableInputs { static_inputs, ..Default::default() }, + ); + } + policy.triggerables_v2 = Some(triggerables_v2); + } + policy +} + async fn execute_component( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -1228,99 +1280,136 @@ async fn execute_component( }; let path = path.to_path(); + let (arc_policy, policy): (Arc, Policy); + let policy_triggerables_default = Default::default(); - let policy = match payload.clone() { + // Two cases here: + // 1. The component is executed from the editor (i.e. in "preview" mode), then: + // - The policy is set to default (in `Viewer` execution mode). + // - The policy triggerables are built by the frontend and retrieved from the request + // payload. + // - In case of inline script, the `RawCode` from the request is pushed as is to the + // job queue. + // 2. Otherwise (i.e. "run" mode): + // - The policy and triggerables are fetched from the database. + // - In case of inline script, if an entry exists in the `app_script` table, push + // an `AppScript` job payload, as in (.1) otherwise. + let (policy, policy_triggerables) = match payload { + // 1. "preview" mode. ExecuteApp { - force_viewer_static_fields: Some(static_fields), - force_viewer_one_of_fields: Some(one_of_fields), + force_viewer_static_fields: Some(static_inputs), + force_viewer_one_of_fields: Some(one_of_inputs), force_viewer_allow_user_resources: Some(allow_user_resources), .. - } => { - let mut hm = HashMap::new(); - - if let Some(path) = payload.path.clone() { - hm.insert( - format!("{}:{path}", payload.component), - PolicyTriggerableInputs { - static_inputs: static_fields, - one_of_inputs: one_of_fields, - allow_user_resources, - }, - ); - } else { - hm.insert( - format!( - "{}:{}", - payload.component, - digest(payload.raw_code.clone().unwrap().content.as_str()) - ), - PolicyTriggerableInputs { - static_inputs: static_fields, - one_of_inputs: one_of_fields, - allow_user_resources, - }, - ); - } - Policy { + } => ( + &Policy { execution_mode: ExecutionMode::Viewer, - triggerables: None, - triggerables_v2: Some(hm), - on_behalf_of: None, - on_behalf_of_email: None, - s3_inputs: None, - } - } + ..Default::default() + }, + &PolicyTriggerableInputs { + static_inputs, + one_of_inputs, + allow_user_resources, + }, + ), + // 2. "run" mode. _ => { - let policy_o = sqlx::query_scalar!( - "SELECT policy from app WHERE path = $1 AND workspace_id = $2", + // Policy is fetched from the database on app `path` and `workspace_id`. + let policy_fut = sqlx::query_scalar!( + "SELECT policy as \"policy: sqlx::types::Json>\" + FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1", path, - &w_id + &w_id, ) .fetch_optional(&db) - .await?; + .map_err(Into::::into) + .map(|policy_o| Result::Ok(not_found_if_none(policy_o?, "App", path)?)) + .map(|policy| Result::Ok(serde_json::from_str(policy?.get())?)) + .map_ok(empty_triggerables); - let policy = not_found_if_none(policy_o, "App", path)?; + // 1. The app `version` is provided: cache the fetched policy. + // 2. Otherwise, always fetch the policy from the database. + let policy = if let Some(id) = payload.version { + let cache = cache::anon!({ u64 => Arc } in "policy" <= 1000); + arc_policy = policy_fut + .map_ok(Arc::new) + .cached(cache, &(id as u64)) + .await?; + &*arc_policy + } else { + policy = policy_fut.await?; + &policy + }; - serde_json::from_value::(policy).map_err(to_anyhow)? + // Compute the path for the triggerables map: + // - flow: `flow/` + // - script: `script/` + // - inline script: `rawscript/` + let path = match &payload { + // flow or script: just use the `payload.path`. + ExecuteApp { path: Some(path), .. } => path, + // inline script: without entry in the `app_script` table. + ExecuteApp { raw_code: Some(raw_code), id: None, .. } => &digest(&raw_code.content), + // inline script: with an entry in the `app_script` table. + ExecuteApp { raw_code: Some(_), id: Some(id), .. } => { + let cache = cache::anon!({ u64 => Arc } in "appscriptpath" <= 10000); + // `id` is unique, cache the result. + &*sqlx::query_scalar!( + "SELECT format('rawscript/%s', code_sha256) as \"path!: String\" + FROM app_script WHERE id = $1 LIMIT 1", + id + ) + .fetch_one(&db) + .map_err(Into::::into) + .map_ok(Arc::new) + .cached(cache, &(*id as u64)) + .await? + } + _ => unreachable!(), + }; + + // Retrieve the triggerables from the policy on `path` or `:`. + let triggerables_v2 = policy + .triggerables_v2 + .as_ref() + .ok_or_else(|| Error::BadRequest(format!("Policy is missing triggerables")))?; + let policy_triggerables = triggerables_v2 + .get(path) // start with `path` in case we can avoid the next` format!`. + .or_else(|| triggerables_v2.get(&format!("{}:{}", payload.component, &path))) + .or(match policy.execution_mode { + ExecutionMode::Viewer => Some(&policy_triggerables_default), + _ => None, + }) + .ok_or_else(|| Error::BadRequest(format!("Path {path} forbidden by policy")))?; + + (policy, policy_triggerables) } }; let (username, permissioned_as, email) = get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; - let (job_payload, (args, job_id), tag) = match payload { - ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => { - let content = &raw_code.content; - let payload = JobPayload::Code(raw_code.clone()); - let path = digest(content); - let args = build_args( - policy, - &component, - path, - args, - opt_authed.as_ref(), - &user_db, - &db, - &w_id, - ) - .await?; - (payload, args, None) - } - ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => { - let (payload, tag) = get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?; - let args = build_args( - policy, - &component, - path.to_string(), - args, - opt_authed.as_ref(), - &user_db, - &db, - &w_id, - ) - .await?; - (payload, args, tag) - } + let (args, job_id) = build_args( + policy, + policy_triggerables, + payload.args, + opt_authed.as_ref(), + &user_db, + &db, + &w_id, + ) + .await?; + + let (job_payload, tag) = match (payload.path, payload.raw_code, payload.id) { + // flow or script: + (Some(path), None, None) => get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?, + // inline script: in "preview" mode or without entry in the `app_script` table. + (None, Some(raw_code), None) => (JobPayload::Code(raw_code), None), + // inline script: in "run" mode and with an entry in the `app_script` table. + (None, Some(RawCode { language, path, cache_ttl, .. }), Some(id)) => ( + JobPayload::AppScript { id: AppScriptId(id), cache_ttl, language, path }, + None, + ), _ => unreachable!(), }; let tx = windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()); @@ -1655,9 +1744,12 @@ async fn exists_app( } async fn build_args( - policy: Policy, - component: &str, - path: String, + policy: &Policy, + PolicyTriggerableInputs { + static_inputs, + one_of_inputs, + allow_user_resources, + }: &PolicyTriggerableInputs, mut args: HashMap>, authed: Option<&ApiAuthed>, user_db: &UserDB, @@ -1665,54 +1757,6 @@ async fn build_args( w_id: &str, ) -> Result<(PushArgsOwned, Option)> { let mut job_id: Option = None; - let key = format!("{}:{}", component, &path); - let (static_inputs, one_of_inputs, allow_user_resources) = match policy { - Policy { triggerables_v2: Some(t), .. } => { - let PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources } = t - .get(&key) - .or_else(|| t.get(&path)) - .map(|x| x.clone()) - .or_else(|| { - if matches!(policy.execution_mode, ExecutionMode::Viewer) { - Some(PolicyTriggerableInputs { - static_inputs: HashMap::new(), - one_of_inputs: HashMap::new(), - allow_user_resources: Vec::new(), - }) - } else { - None - } - }) - .ok_or_else(|| { - Error::BadRequest(format!("path {} is not allowed in the app policy", path)) - })?; - - (static_inputs, one_of_inputs, allow_user_resources) - } - Policy { triggerables: Some(t), .. } => { - let static_inputs = t - .get(&key) - .or_else(|| t.get(&path)) - .map(|x| x.clone()) - .or_else(|| { - if matches!(policy.execution_mode, ExecutionMode::Viewer) { - Some(HashMap::new()) - } else { - None - } - }) - .ok_or_else(|| { - Error::BadRequest(format!("path {} is not allowed in the app policy", path)) - })?; - - (static_inputs, HashMap::new(), Vec::new()) - } - _ => Err(Error::BadRequest(format!( - "Policy is missing triggerables for {}", - key - )))?, - }; - let mut safe_args = HashMap::>::new(); // tracing::error!("{:?}", allow_user_resources); @@ -1761,16 +1805,16 @@ async fn build_args( } for (k, v) in one_of_inputs { - if safe_args.contains_key(&k) { + if safe_args.contains_key(k) { continue; } - if let Some(arg_val) = args.get(&k) { + if let Some(arg_val) = args.get(k) { let arg_str = arg_val.get(); let options_str_vec = v.iter().map(|x| x.get()).collect::>(); if options_str_vec.contains(&arg_str) { safe_args.insert(k.to_string(), arg_val.clone()); - args.remove(&k); + args.remove(k); continue; } @@ -1781,7 +1825,7 @@ async fn build_args( .all(|x| options_str_vec.contains(&x.get())) { safe_args.insert(k.to_string(), arg_val.clone()); - args.remove(&k); + args.remove(k); continue; } } diff --git a/backend/windmill-common/src/apps.rs b/backend/windmill-common/src/apps.rs index acbf720f57..2948fd7cb7 100644 --- a/backend/windmill-common/src/apps.rs +++ b/backend/windmill-common/src/apps.rs @@ -6,7 +6,18 @@ * LICENSE-AGPL for a copy of the license. */ -use serde::Deserialize; +use serde::{Deserialize, Serialize}; + +/// Id in the `app_script` table. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)] +#[serde(transparent)] +pub struct AppScriptId(pub i64); + +impl Into for AppScriptId { + fn into(self) -> u64 { + self.0 as u64 + } +} #[derive(Deserialize)] pub struct ListAppQuery { diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 9a81c7dace..45fdd0bd82 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -394,6 +394,96 @@ pub mod script { } } +pub mod app { + use super::*; + use crate::apps::AppScriptId; + + make_static! { + /// App scripts cache. + /// FIXME: Use `Arc` for cheap cloning. + static ref CACHE: { AppScriptId => Val } in "app" <= 1000; + } + + /// App app script cache value. + #[derive(Debug, Clone, Default)] + pub struct Val { + pub lock: Option, + pub code: String, + } + + /// Fetch the app script referenced by `id` from the cache. + /// If not present, import from the file-system cache or fetch it from the database and write + /// it to the file system and cache. + /// This should be preferred over fetching the database directly. + pub async fn fetch_script( + e: impl PgExecutor<'_>, + id: AppScriptId, + ) -> error::Result<(Option, String)> { + // If not present, `get_or_insert_async` will lock the key until the future completes, + // so only one thread will be able to fetch the data from the database and write it to + // the file system and cache, hence no race on the file system. + CACHE + .get_or_insert_async(&id, async { + sqlx::query!( + "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", + id.0, + ) + .fetch_one(e) + .await + .map_err(Into::into) + .map(|r| Val { + lock: r + .lock + .and_then(|x| if x.is_empty() { None } else { Some(x) }), + code: r.code, + }) + }) + .await + .map(|Val { lock, code }| (lock, code)) + } + + // ---------------------------------------------------------------------------------------------- + // impl `fs::Bundle` for `Val`. + + #[derive(Copy, Clone)] + pub enum Item { + Lock, + Code, + } + + impl fs::Item for Item { + fn path(&self, root: &Path) -> PathBuf { + match self { + Item::Lock => root.join("lock.txt"), + Item::Code => root.join("code.txt"), + } + } + } + + impl fs::Bundle for Val { + type Item = Item; + + fn items() -> &'static [Self::Item] { + &[Item::Lock, Item::Code] + } + + fn import(&mut self, item: Self::Item, data: Vec) -> error::Result<()> { + match item { + Item::Lock => self.lock = Some(String::from_utf8(data)?), + Item::Code => self.code = String::from_utf8(data)?, + } + Ok(()) + } + + fn export(&self, item: Self::Item) -> error::Result>> { + match item { + Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())), + Item::Code => Ok(Some(self.code.as_bytes().to_vec())), + } + } + } +} + mod fs { use super::*; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 5b687b54fb..c8f8470b69 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -14,6 +14,7 @@ pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; pub const PREPROCESSOR_FAKE_ENTRYPOINT: &str = "__WM_PREPROCESSOR"; use crate::{ + apps::AppScriptId, error::{self, to_anyhow, Error}, flow_status::{FlowStatus, RestartedFrom}, flows::{FlowNodeId, FlowValue, Retry}, @@ -41,6 +42,7 @@ pub enum JobKind { DeploymentCallback, FlowScript, FlowNode, + AppScript, } #[derive(sqlx::FromRow, Debug, Serialize, Clone)] @@ -278,6 +280,12 @@ pub enum JobPayload { id: FlowNodeId, // flow_node(id). path: String, // flow node inner path (e.g. `outer/branchall-42`). }, + AppScript { + id: AppScriptId, // app_script(id). + path: Option, + language: ScriptLang, + cache_ttl: Option, + }, Code(RawCode), Dependencies { path: String, diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 1e516f4821..c26a92872f 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -3228,7 +3228,27 @@ pub async fn push<'c, 'd>( None, None, ) - } + }, + JobPayload::AppScript { + id, // app_script(id). + path, + language, + cache_ttl, + } => ( + Some(id.0), + path, + None, + JobKind::AppScript, + None, + None, + Some(language), + None, + None, + None, + cache_ttl, + None, + None, + ), JobPayload::ScriptHub { path } => { if path == "hub/7771/slack" || path == "hub/7836/slack" { permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string(); @@ -3989,6 +4009,7 @@ pub async fn push<'c, 'd>( JobKind::DeploymentCallback => "jobs.run.deployment_callback", JobKind::FlowScript => "jobs.run.flow_script", JobKind::FlowNode => "jobs.run.flow_node", + JobKind::AppScript => "jobs.run.app_script", }; let audit_author = if format!("u/{user}") != permissioned_as && user != permissioned_as { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 47f2fb393f..238fd6a951 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -7,6 +7,7 @@ */ use windmill_common::{ + apps::AppScriptId, auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET}, scripts::PREVIEW_IS_TAR_CODEBASE_HASH, utils::WarnAfterExt, @@ -2311,6 +2312,20 @@ async fn handle_code_execution_job( codebase: None, } } + JobKind::AppScript => { + let (lockfile, content) = cache::app::fetch_script( + db, + AppScriptId(job.script_hash.unwrap_or(ScriptHash(0)).0), + ) + .await?; + ContentReqLangEnvs { + content, + lockfile, + language: job.language.to_owned(), + envs: None, + codebase: None, + } + } JobKind::DeploymentCallback => { get_script_content_by_path(job.script_path.clone(), &job.workspace_id, db).await? } diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index ec739a88a2..ff1216da65 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -15,6 +15,7 @@ use windmill_common::jobs::JobPayload; use windmill_common::scripts::ScriptHash; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file}; use windmill_common::{ + apps::AppScriptId, error::{self, to_anyhow}, flows::{add_virtual_items_if_necessary, FlowValue}, jobs::QueuedJob, @@ -648,7 +649,7 @@ pub async fn handle_flow_dependency_job( // Compute a lite version of the flow value (`RawScript` => `FlowScript`). let mut value_lite = flow.clone(); - tx = reduce( + tx = reduce_flow( tx, &mut value_lite.modules, &job_path, @@ -1053,6 +1054,41 @@ async fn insert_flow_node<'c>( Ok((tx, FlowNodeId(id))) } +async fn insert_app_script( + db: &sqlx::Pool, + app: i64, + code: String, + lock: Option, +) -> Result { + let code_sha256 = format!("{:x}", sha2::Sha256::digest(&code)); + let hash = { + let mut hasher = sha2::Sha256::new(); + hasher.update(app.to_le_bytes()); + hasher.update(&code_sha256); + hasher.update(lock.as_ref().unwrap_or(&Default::default())); + format!("{:x}", hasher.finalize()) + }; + + // Insert the app script if it doesn't exist. + sqlx::query_scalar!( + r#" + INSERT INTO app_script (app, hash, lock, code, code_sha256) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id + RETURNING id + "#, + app, + hash, + lock, + code, + code_sha256 + ) + .fetch_one(db) + .await + .map(AppScriptId) + .map_err(Into::into) +} + async fn insert_flow_modules<'c>( mut tx: sqlx::Transaction<'c, sqlx::Postgres>, path: &str, @@ -1062,7 +1098,7 @@ async fn insert_flow_modules<'c>( modules: &mut Vec, modules_node: &mut Option, ) -> Result> { - tx = Box::pin(reduce( + tx = Box::pin(reduce_flow( tx, modules, path, @@ -1094,7 +1130,7 @@ async fn insert_flow_modules<'c>( Ok(tx) } -async fn reduce<'c>( +async fn reduce_flow<'c>( mut tx: sqlx::Transaction<'c, sqlx::Postgres>, modules: &mut Vec, path: &str, @@ -1107,7 +1143,7 @@ async fn reduce<'c>( let mut val = serde_json::from_str::(module.value.get()).map_err(|err| { Error::InternalErr(format!( - "reduce: Failed to parse flow module value: {}", + "reduce_flow: Failed to parse flow module value: {}", err )) })?; @@ -1203,6 +1239,41 @@ async fn reduce<'c>( Ok(tx) } +async fn reduce_app(db: &sqlx::Pool, value: &mut Value, app: i64) -> Result<()> { + match value { + Value::Object(object) => { + if let Some(Value::Object(script)) = object.get_mut("inlineScript") { + // replace `content` with an empty string: + let Some(Value::String(code)) = script.get_mut("content").map(std::mem::take) + else { + return Err(error::Error::InternalErr( + "Missing `content` in inlineScript".to_string(), + )); + }; + // remove `lock`: + let lock = script.remove("lock").and_then(|x| match x { + Value::String(s) => Some(s), + _ => None, + }); + let id = insert_app_script(db, app, code, lock).await?; + // insert the `id` into the `script` object: + script.insert("id".to_string(), json!(id.0)); + } else { + for (_, value) in object { + Box::pin(reduce_app(db, value, app)).await?; + } + } + } + Value::Array(array) => { + for value in array { + Box::pin(reduce_app(db, value, app)).await?; + } + } + _ => {} + } + Ok(()) +} + fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool { if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative { let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content); @@ -1388,11 +1459,12 @@ pub async fn handle_app_dependency_job( .clone() .ok_or_else(|| Error::InternalErr("App Dependency requires script hash".to_owned()))? .0; - let value = sqlx::query_scalar!("SELECT value FROM app_version WHERE id = $1", id) + let record = sqlx::query!("SELECT app_id, value FROM app_version WHERE id = $1", id) .fetch_optional(db) - .await?; + .await? + .map(|record| (record.app_id, record.value)); - if let Some(value) = value { + if let Some((app_id, value)) = record { let value = lock_modules_app( value, job, @@ -1409,6 +1481,21 @@ pub async fn handle_app_dependency_job( ) .await?; + // Compute a lite version of the app value (w/ `inlineScript.{lock,code}`). + let mut value_lite = value.clone(); + reduce_app(db, &mut value_lite, app_id).await?; + if let Value::Object(object) = &mut value_lite { + object.insert("version".to_string(), json!(id)); + } + sqlx::query!( + "INSERT INTO app_version_lite (id, value) VALUES ($1, $2) + ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value", + id, + sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json>, + ) + .execute(db) + .await?; + // Re-check cancelation to ensure we don't accidentially override an app. if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id) .fetch_optional(db) diff --git a/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte b/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte index 9119282752..b135bc680a 100644 --- a/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte +++ b/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte @@ -377,11 +377,14 @@ : runnable if (inlineScript) { + if (inlineScript.id !== undefined) { + requestBody['id'] = inlineScript.id + } requestBody['raw_code'] = { - content: inlineScript.content, + content: inlineScript.id === undefined ? inlineScript.content : '', language: inlineScript.language ?? '', path: inlineScript.path, - lock: inlineScript.lock, + lock: inlineScript.id === undefined ? inlineScript.lock : undefined, cache_ttl: inlineScript.cache_ttl } } @@ -390,6 +393,10 @@ requestBody['path'] = runType !== 'hubscript' ? `${runType}/${path}` : `script/${path}` } + if ($app.version !== undefined) { + requestBody['version'] = $app.version + } + const uuid = await AppService.executeComponent({ workspace, path: defaultIfEmptyString($appPath, `u/${$userStore?.username ?? 'unknown'}/newapp`), diff --git a/frontend/src/lib/components/apps/types.ts b/frontend/src/lib/components/apps/types.ts index 5e6e8f00fa..7debc557f1 100644 --- a/frontend/src/lib/components/apps/types.ts +++ b/frontend/src/lib/components/apps/types.ts @@ -117,6 +117,7 @@ export type InlineScript = { cache_ttl?: number refreshOn?: { id: string; key: string }[] suggestedRefreshOn?: { id: string; key: string }[] + id?: number } export type AppCssItemName = 'viewer' | 'grid' | AppComponent['type'] @@ -163,6 +164,7 @@ export type App = { theme: AppTheme | undefined hideLegacyTopBar?: boolean | undefined mobileViewOnSmallerScreens?: boolean | undefined + version?: number } export type ConnectingInput = { diff --git a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte index 1167ab1643..09bec77e67 100644 --- a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte @@ -17,7 +17,7 @@ let can_write = false async function loadApp() { - app = await AppService.getAppByPath({ workspace: $workspaceStore!, path: $page.params.path }) + app = await AppService.getAppLiteByPath({ workspace: $workspaceStore!, path: $page.params.path }) can_write = canWrite(app?.path, app?.extra_perms!, $userStore) } From 691ef6468823c99ab129c53ae699de1a91f77c97 Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Fri, 6 Dec 2024 15:11:57 +0100 Subject: [PATCH 38/48] feat(frontend): render new job kinds (#4864) --- backend/windmill-api/openapi-deref.yaml | 6 ++++++ backend/windmill-api/openapi.yaml | 4 ++++ backend/windmill-worker/src/windmill-client.js | 6 ++++++ frontend/src/lib/components/runs/JobLoader.svelte | 7 +++++-- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 0e7502d6cd..c8893f824c 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -8709,6 +8709,9 @@ paths: - identity - deploymentcallback - singlescriptflow + - flowscript + - flownode + - appscript schedule_path: type: string permissioned_as: @@ -9267,6 +9270,9 @@ paths: - identity - deploymentcallback - singlescriptflow + - flowscript + - flownode + - appscript schedule_path: type: string permissioned_as: diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ccab668051..0bce856613 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10983,6 +10983,8 @@ components: "deploymentcallback", "singlescriptflow", "flowscript", + "flownode", + "appscript", ] schedule_path: type: string @@ -11103,6 +11105,8 @@ components: "deploymentcallback", "singlescriptflow", "flowscript", + "flownode", + "appscript", ] schedule_path: type: string diff --git a/backend/windmill-worker/src/windmill-client.js b/backend/windmill-worker/src/windmill-client.js index 64281fede9..3ae15e7453 100644 --- a/backend/windmill-worker/src/windmill-client.js +++ b/backend/windmill-worker/src/windmill-client.js @@ -558,6 +558,9 @@ var $QueuedJob = { "identity", "deploymentcallback", "singlescriptflow", + "flowscript", + "flownode", + "appscript", ], }, schedule_path: { @@ -697,6 +700,9 @@ var $CompletedJob = { "identity", "deploymentcallback", "singlescriptflow", + "flowscript", + "flownode", + "appscript", ], }, schedule_path: { diff --git a/frontend/src/lib/components/runs/JobLoader.svelte b/frontend/src/lib/components/runs/JobLoader.svelte index d58a9d0af3..49d14f46a0 100644 --- a/frontend/src/lib/components/runs/JobLoader.svelte +++ b/frontend/src/lib/components/runs/JobLoader.svelte @@ -100,7 +100,10 @@ 'appdependencies', 'preview', 'flowpreview', - 'script_hub' + 'script_hub', + 'flowscript', + 'flownode', + 'appscript', ] return kinds.join(',') } else if (jobKindsCat == 'dependencies') { @@ -117,7 +120,7 @@ let kinds: CompletedJob['job_kind'][] = ['deploymentcallback'] return kinds.join(',') } else { - let kinds: CompletedJob['job_kind'][] = ['script', 'flow'] + let kinds: CompletedJob['job_kind'][] = ['script', 'flow', 'flowscript', 'flownode', 'appscript'] return kinds.join(',') } } From 8e9cb9c4d346281d116ce1bacc14017aacd5c366 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Fri, 6 Dec 2024 16:52:03 +0100 Subject: [PATCH 39/48] ee compute units telemetry (#4865) * ee compute units telemetry * update ee ref --- ...29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json} | 10 ++++++++-- backend/ee-repo-ref.txt | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) rename backend/.sqlx/{query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json => query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json} (74%) diff --git a/backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json b/backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json similarity index 74% rename from backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json rename to backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json index 7e1f008dba..1521b9fd2f 100644 --- a/backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json +++ b/backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", + "query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at, worker_group FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "started_at", "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "worker_group", + "type_info": "Varchar" } ], "parameters": { @@ -43,8 +48,9 @@ true, true, false, + false, false ] }, - "hash": "a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7" + "hash": "6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c7e750d0f4..6bb5c4686f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -8606d98a692d11b09a387c5efbd6b4335c533fd3 \ No newline at end of file +5066da602260334767186e69ae5b6821feca0c71 \ No newline at end of file From 6d047449e200d51f060d4107ff5ad0e10af99a66 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 7 Dec 2024 16:29:20 +0100 Subject: [PATCH 40/48] feat: add otlp support (#4869) * all * all * add otel * add otel * update docker-image * update * update * update * update * update * update * update * update * update * update * update --- .github/workflows/docker-image.yml | 4 +- backend/Cargo.lock | 188 ++- backend/Cargo.toml | 9 +- backend/ee-repo-ref.txt | 2 +- backend/src/main.rs | 147 ++- backend/src/monitor.rs | 66 +- backend/tests/worker.rs | 5 +- backend/windmill-api/src/tracing_init.rs | 18 +- backend/windmill-common/Cargo.toml | 8 + .../windmill-common/src/global_settings.rs | 6 +- backend/windmill-common/src/lib.rs | 10 +- backend/windmill-common/src/otel_ee.rs | 54 + backend/windmill-common/src/tracing_init.rs | 94 +- backend/windmill-queue/Cargo.toml | 3 +- backend/windmill-queue/src/jobs.rs | 8 - backend/windmill-worker/Cargo.toml | 4 + backend/windmill-worker/src/common.rs | 10 +- backend/windmill-worker/src/handle_child.rs | 15 +- .../windmill-worker/src/result_processor.rs | 119 +- backend/windmill-worker/src/worker.rs | 153 ++- backend/windmill-worker/src/worker_flow.rs | 87 +- .../src/lib/components/AuthSettings.svelte | 255 ++++ .../src/lib/components/InstanceSetting.svelte | 783 ++++++++++++ .../lib/components/InstanceSettings.svelte | 1103 ++--------------- .../src/lib/components/OAuthSetting.svelte | 7 +- .../lib/components/OauthExtraParams.svelte | 2 +- .../src/lib/components/OauthScopes.svelte | 4 +- .../ObjectStoreConfigSettings.svelte | 269 ++-- .../lib/components/SuperadminSettings.svelte | 38 +- .../src/lib/components/instanceSettings.ts | 194 +-- 30 files changed, 2133 insertions(+), 1532 deletions(-) create mode 100644 backend/windmill-common/src/otel_ee.rs create mode 100644 frontend/src/lib/components/AuthSettings.svelte create mode 100644 frontend/src/lib/components/InstanceSetting.svelte diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 42cc21264b..f1446210cc 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -138,7 +138,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} @@ -200,7 +200,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel PYTHON_IMAGE=python:3.12.2-slim-bookworm tags: | ${{ steps.meta-ee-public-py312.outputs.tags }} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index dac85d4f09..e003c2300a 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -4173,6 +4173,19 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper 1.5.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -4809,7 +4822,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "674883a98273598ac3aad4301724c56734bea90574c5033af067e8f9fb5eb399" dependencies = [ - "prost", + "prost 0.12.6", "prost-types", ] @@ -5637,6 +5650,91 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3cebff57f7dbd1255b44d8bddc2cebeb0ea677dbaa2e25a3070a91b318f660" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror 1.0.69", +] + +[[package]] +name = "opentelemetry-appender-tracing" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5feffc321035ad94088a7e5333abb4d84a8726e54a802e736ce9dd7237e85b" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +dependencies = [ + "async-trait", + "futures-core", + "http 1.2.0", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost 0.13.3", + "thiserror 1.0.69", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost 0.13.3", + "tonic", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" + +[[package]] +name = "opentelemetry_sdk" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b742c1cae4693792cc564e58d75a2a0ba29421a34a85b50da92efa89ecb2bc" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "percent-encoding", + "rand 0.8.5", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -6264,7 +6362,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.6", +] + +[[package]] +name = "prost" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +dependencies = [ + "bytes", + "prost-derive 0.13.3", ] [[package]] @@ -6280,13 +6388,26 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "prost-derive" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +dependencies = [ + "anyhow", + "itertools 0.13.0", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "prost-types" version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ - "prost", + "prost 0.12.6", ] [[package]] @@ -9448,6 +9569,36 @@ dependencies = [ "winnow 0.6.20", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2 0.4.7", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.5.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.3", + "socket2", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "toolchain_find" version = "0.4.0" @@ -9469,11 +9620,16 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand 0.8.5", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -9651,6 +9807,24 @@ dependencies = [ "url", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log 0.2.0", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -10644,6 +10818,11 @@ dependencies = [ "magic-crypt", "mail-send", "object_store", + "opentelemetry", + "opentelemetry-appender-tracing", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", "pin-project-lite", "prometheus", "quick_cache", @@ -10662,6 +10841,7 @@ dependencies = [ "tracing-appender", "tracing-flame", "tracing-loki", + "tracing-opentelemetry", "tracing-subscriber", "uuid 1.11.0", "windmill-macros", @@ -10897,6 +11077,7 @@ dependencies = [ "hmac", "itertools 0.13.0", "lazy_static", + "opentelemetry", "prometheus", "regex", "reqwest 0.12.9", @@ -10961,6 +11142,7 @@ dependencies = [ "object_store", "once_cell", "openidconnect", + "opentelemetry", "pem 3.0.4", "postgres-native-tls", "prometheus", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 3f591dfd84..4646af9f5e 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -63,6 +63,7 @@ tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"] sqlx = ["windmill-worker/sqlx"] deno_core = ["windmill-worker/deno_core", "dep:deno_core"] kafka = ["windmill-api/kafka"] +otel = ["windmill-common/otel", "windmill-worker/otel"] [dependencies] anyhow.workspace = true @@ -95,7 +96,6 @@ deno_core = { workspace = true, optional = true } object_store = { workspace = true, optional = true } quote.workspace = true - [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } tikv-jemalloc-sys = { optional = true, workspace = true } @@ -279,6 +279,13 @@ tar = "^0" http = "^1" async-stream = "^0" +opentelemetry = "0.27.0" +tracing-opentelemetry = "0.28.0" +opentelemetry_sdk = { version = "*", features = ["rt-tokio"] } +opentelemetry-otlp = "0.27.0" +opentelemetry-appender-tracing = "0.27.0" +opentelemetry-semantic-conventions = "*" + tikv-jemallocator = { version = "0.5" } tikv-jemalloc-sys = { version = "^0.5" } tikv-jemalloc-ctl = { version = "^0.5" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 6bb5c4686f..b5181c39f2 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -5066da602260334767186e69ae5b6821feca0c71 \ No newline at end of file +ad89b5a1566159490eceb59c3e5c9416d9aa855c \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index d403ad212a..80edff1ac9 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -8,7 +8,7 @@ use anyhow::Context; use monitor::{ - reload_delete_logs_periodically_setting, reload_indexer_config, + load_otel, reload_delete_logs_periodically_setting, reload_indexer_config, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store, }; @@ -37,7 +37,7 @@ use windmill_common::{ EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, - OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, @@ -221,11 +221,66 @@ async fn windmill_main() -> anyhow::Result<()> { let hostname = hostname(); - #[cfg(not(feature = "flamegraph"))] - let _guard = windmill_common::tracing_init::initialize_tracing(&hostname); + let mut enable_standalone_indexer: bool = false; + + let mode = std::env::var("MODE") + .map(|x| x.to_lowercase()) + .map(|x| { + if &x == "server" { + println!("Binary is in 'server' mode"); + Mode::Server + } else if &x == "worker" { + tracing::info!("Binary is in 'worker' mode"); + #[cfg(windows)] + { + println!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL"); + } + Mode::Worker + } else if &x == "agent" { + println!("Binary is in 'agent' mode"); + if std::env::var("BASE_INTERNAL_URL").is_err() { + panic!("BASE_INTERNAL_URL is required in agent mode") + } + if std::env::var("JOB_TOKEN").is_err() { + println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table") + } + + #[cfg(not(feature = "enterprise"))] + { + panic!("Agent mode is only available in the EE, ignoring..."); + } + #[cfg(feature = "enterprise")] + Mode::Agent + } else if &x == "indexer" { + tracing::info!("Binary is in 'indexer' mode"); + #[cfg(not(feature = "tantivy"))] + { + eprintln!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features."); + panic!("Indexer mode requires compiling with the tantivy feature flag."); + } + #[cfg(feature = "tantivy")] + Mode::Indexer + } else if &x == "standalone+search"{ + enable_standalone_indexer = true; + println!("Binary is in 'standalone' mode with search enabled"); + Mode::Standalone + } + else { + if &x != "standalone" { + eprintln!("mode not recognized, defaulting to standalone: {x}"); + } else { + println!("Binary is in 'standalone' mode"); + } + Mode::Standalone + } + }) + .unwrap_or_else(|_| { + tracing::info!("Mode not specified, defaulting to standalone"); + Mode::Standalone + }); #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] - tracing::info!("jemalloc enabled"); + println!("jemalloc enabled"); #[cfg(feature = "flamegraph")] let _guard = windmill_common::tracing_init::setup_flamegraph(); @@ -236,13 +291,13 @@ async fn windmill_main() -> anyhow::Result<()> { "cache" => { #[cfg(feature = "embedding")] { - tracing::info!("Caching embedding model..."); + println!("Caching embedding model..."); windmill_api::embeddings::ModelInstance::load_model_files().await?; - tracing::info!("Cached embedding model"); + println!("Cached embedding model"); } #[cfg(not(feature = "embedding"))] { - tracing::warn!("Embeddings are not enabled, ignoring..."); + println!("Embeddings are not enabled, ignoring..."); } cache_hub_scripts(std::env::args().nth(2)).await?; @@ -256,64 +311,6 @@ async fn windmill_main() -> anyhow::Result<()> { _ => {} } - let mut enable_standalone_indexer: bool = false; - - let mode = std::env::var("MODE") - .map(|x| x.to_lowercase()) - .map(|x| { - if &x == "server" { - tracing::info!("Binary is in 'server' mode"); - Mode::Server - } else if &x == "worker" { - tracing::info!("Binary is in 'worker' mode"); - #[cfg(windows)] - { - tracing::warn!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL"); - } - Mode::Worker - } else if &x == "agent" { - tracing::info!("Binary is in 'agent' mode"); - if std::env::var("BASE_INTERNAL_URL").is_err() { - panic!("BASE_INTERNAL_URL is required in agent mode") - } - if std::env::var("JOB_TOKEN").is_err() { - tracing::warn!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table") - } - - #[cfg(not(feature = "enterprise"))] - { - panic!("Agent mode is only available in the EE, ignoring..."); - } - #[cfg(feature = "enterprise")] - Mode::Agent - } else if &x == "indexer" { - tracing::info!("Binary is in 'indexer' mode"); - #[cfg(not(feature = "tantivy"))] - { - tracing::error!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features."); - panic!("Indexer mode requires compiling with the tantivy feature flag."); - } - #[cfg(feature = "tantivy")] - Mode::Indexer - } else if &x == "standalone+search"{ - enable_standalone_indexer = true; - tracing::info!("Binary is in 'standalone' mode with search enabled"); - Mode::Standalone - } - else { - if &x != "standalone" { - tracing::error!("mode not recognized, defaulting to standalone: {x}"); - } else { - tracing::info!("Binary is in 'standalone' mode"); - } - Mode::Standalone - } - }) - .unwrap_or_else(|_| { - tracing::info!("Mode not specified, defaulting to standalone"); - Mode::Standalone - }); - #[allow(unused_mut)] let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer { 0 @@ -325,7 +322,7 @@ async fn windmill_main() -> anyhow::Result<()> { }; if num_workers > 1 { - tracing::warn!( + println!( "We STRONGLY recommend using at most 1 worker per container, use at your own risks" ); } @@ -347,10 +344,15 @@ async fn windmill_main() -> anyhow::Result<()> { IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)) }; - tracing::info!("Connecting to database..."); + println!("Connecting to database..."); let db = windmill_common::connect_db(server_mode, indexer_mode).await?; + + load_otel(&db).await; tracing::info!("Database connected"); + #[cfg(not(feature = "flamegraph"))] + let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode); + let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await; tracing::info!( @@ -776,6 +778,15 @@ Windmill Community Edition {GIT_VERSION} tracing::error!(error = %e, "Could not reload debug metrics setting"); } }, + OTEL_SETTING => { + tracing::info!("OTEL setting changed, restarting"); + // we wait a bit randomly to avoid having all servers and workers shutdown at same time + let rd_delay = rand::thread_rng().gen_range(0..4); + tokio::time::sleep(Duration::from_secs(rd_delay)).await; + if let Err(e) = tx.send(()) { + tracing::error!(error = %e, "Could not send killpill"); + } + }, REQUEST_SIZE_LIMIT_SETTING => { if server_mode { tracing::info!("Request limit size change detected, killing server expecting to be restarted"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index df3b695168..262927cce3 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -12,7 +12,7 @@ use std::{ use chrono::{NaiveDateTime, Utc}; use futures::{stream::FuturesUnordered, StreamExt}; -use serde::de::DeserializeOwned; +use serde::{de::DeserializeOwned, Deserializer}; use sqlx::{Pool, Postgres}; use tokio::{ join, @@ -40,7 +40,7 @@ use windmill_common::{ EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, - PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, @@ -58,7 +58,8 @@ use windmill_common::{ }, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, - MONITOR_LOGS_ON_OBJECT_STORE, SERVICE_LOG_RETENTION_SECS, + MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, + SERVICE_LOG_RETENTION_SECS, }; use windmill_queue::cancel_job; use windmill_worker::{ @@ -199,6 +200,65 @@ pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> { Ok(()) } +fn empty_string_as_none<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let option = as serde::Deserialize>::deserialize(deserializer)?; + Ok(option.filter(|s| !s.is_empty())) +} + +#[derive(serde::Deserialize)] +struct OtelSetting { + metrics_enabled: Option, + logs_enabled: Option, + tracing_enabled: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_endpoint: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_headers: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_protocol: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_compression: Option, +} + +pub async fn load_otel(db: &DB) { + let otel = load_value_from_global_settings(db, OTEL_SETTING).await; + if let Ok(v) = otel { + if let Some(v) = v { + let deser = serde_json::from_value::(v); + if let Ok(o) = deser { + let metrics_enabled = o.metrics_enabled.unwrap_or(false); + let logs_enabled = o.logs_enabled.unwrap_or(false); + let tracing_enabled = o.tracing_enabled.unwrap_or(false); + + OTEL_METRICS_ENABLED.store(metrics_enabled, Ordering::Relaxed); + OTEL_LOGS_ENABLED.store(logs_enabled, Ordering::Relaxed); + OTEL_TRACING_ENABLED.store(tracing_enabled, Ordering::Relaxed); + if let Some(endpoint) = o.otel_exporter_otlp_endpoint.as_ref() { + std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint); + } + if let Some(headers) = o.otel_exporter_otlp_headers.as_ref() { + std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", headers); + } + if let Some(protocol) = o.otel_exporter_otlp_protocol { + std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", protocol); + } + if let Some(compression) = o.otel_exporter_otlp_compression { + std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", compression); + } + tracing::info!("OTEL settings loaded: tracing ({tracing_enabled}), logs ({logs_enabled}), metrics ({metrics_enabled}), endpoint ({:?}), headers defined: ({})", + o.otel_exporter_otlp_endpoint, o.otel_exporter_otlp_headers.is_some()); + } else { + tracing::error!("Error deserializing otel settings"); + } + } + } else { + tracing::error!("Error loading otel settings: {}", otel.unwrap_err()); + } +} + pub async fn load_tag_per_workspace_enabled(db: &DB) -> error::Result<()> { let metrics_enabled = load_value_from_global_settings(db, DEFAULT_TAGS_PER_WORKSPACE_SETTING).await; diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index a229450651..2aebf6f5e8 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -77,7 +77,10 @@ async fn initialize_tracing() { static ONCE: Once = Once::new(); ONCE.call_once(|| { - let _ = windmill_common::tracing_init::initialize_tracing("test"); + let _ = windmill_common::tracing_init::initialize_tracing( + "test", + &windmill_common::utils::Mode::Standalone, + ); }); } diff --git a/backend/windmill-api/src/tracing_init.rs b/backend/windmill-api/src/tracing_init.rs index 60dab73810..c12cbe3192 100644 --- a/backend/windmill-api/src/tracing_init.rs +++ b/backend/windmill-api/src/tracing_init.rs @@ -29,11 +29,19 @@ impl OnResponse for MyOnResponse { _span: &tracing::Span, ) { if *LOG_REQUESTS { - tracing::info!( - latency = latency.as_millis(), - status = response.status().as_u16(), - "response" - ) + if response.status().is_success() { + tracing::info!( + latency = latency.as_millis(), + status = response.status().as_u16(), + "response" + ) + } else { + tracing::error!( + latency = latency.as_millis(), + status = response.status().as_u16(), + "response" + ) + } } } } diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index b56b9a39cf..5b9a74c361 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -13,6 +13,7 @@ flamegraph = ["dep:tracing-flame"] loki = ["dep:tracing-loki"] benchmark = [] parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"] +otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", "dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing"] [lib] name = "windmill_common" @@ -64,5 +65,12 @@ croner = "2.0.6" quick_cache.workspace = true pin-project-lite.workspace = true +opentelemetry-semantic-conventions = { workspace = true, optional = true } +opentelemetry-otlp = { workspace = true, optional = true } +opentelemetry_sdk = { workspace = true, optional = true } +opentelemetry = { workspace = true, optional = true } +tracing-opentelemetry = { workspace = true, optional = true } +opentelemetry-appender-tracing = { workspace = true, optional = true } + [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = { optional = true, workspace = true } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 7b932d8f50..9dbfd03b0e 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -35,8 +35,9 @@ pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; +pub const OTEL_SETTING: &str = "otel"; -pub const ENV_SETTINGS: [&str; 51] = [ +pub const ENV_SETTINGS: [&str; 54] = [ "DISABLE_NSJAIL", "MODE", "NUM_WORKERS", @@ -88,4 +89,7 @@ pub const ENV_SETTINGS: [&str; 51] = [ "WORKER_GROUP", "SAML_METADATA", "INSTANCE_IS_DEV", + "OTEL_METRICS", + "OTEL_TRACING", + "OTEL_LOGS", ]; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index ad47b198af..4140f6e58b 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -36,20 +36,20 @@ pub mod job_s3_helpers_ee; pub mod jobs; pub mod more_serde; pub mod oauth2; +pub mod otel_ee; pub mod queue; pub mod s3_helpers; pub mod schedule; pub mod scripts; pub mod server; pub mod stats_ee; +pub mod tracing_init; pub mod users; pub mod utils; pub mod variables; pub mod worker; pub mod workspaces; -pub mod tracing_init; - pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50; pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5; pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; @@ -87,6 +87,12 @@ lazy_static::lazy_static! { .unwrap_or_else(|| SocketAddr::from(([0, 0, 0, 0], *METRICS_PORT))); pub static ref METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("METRICS_PORT").is_ok() || std::env::var("METRICS_ADDR").is_ok()); + + pub static ref OTEL_METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_METRICS").is_ok()); + pub static ref OTEL_TRACING_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_TRACING").is_ok()); + pub static ref OTEL_LOGS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_LOGS").is_ok()); + + pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false); pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false); diff --git a/backend/windmill-common/src/otel_ee.rs b/backend/windmill-common/src/otel_ee.rs new file mode 100644 index 0000000000..f310885c0b --- /dev/null +++ b/backend/windmill-common/src/otel_ee.rs @@ -0,0 +1,54 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use crate::{jobs::QueuedJob, utils::Mode}; +use uuid::Uuid; + +pub fn set_span_parent(_span: &tracing::Span, _rj: &Uuid) {} + +#[cfg(not(all(feature = "otel", feature = "enterprise")))] +pub(crate) type OtelProvider = Option<()>; + +#[cfg(all(feature = "otel", feature = "enterprise"))] +pub(crate) type OtelProvider = Option; + +#[cfg(not(feature = "otel"))] +pub fn otel_ctx() -> () {} + +#[cfg(feature = "otel")] +#[inline(always)] +pub fn otel_ctx() -> opentelemetry::Context { + opentelemetry::Context::current() +} + +#[cfg(not(feature = "otel"))] +impl FutureExt for T {} + +#[cfg(not(feature = "otel"))] +pub trait FutureExt: Sized { + fn with_context(self, _otel_cx: ()) -> Self { + self + } +} + +use tracing_subscriber::EnvFilter; + +pub(crate) fn init_logs_bridge(_mode: &Mode) -> Option { + None +} + +#[cfg(all(feature = "otel", feature = "enterprise"))] +pub(crate) fn init_otlp_tracer(_mode: &Mode) -> Option { + None +} + +pub(crate) fn init_meter_provider(_mode: &Mode) -> OtelProvider { + None +} + +pub fn add_root_flow_job_to_otlp(_queued_job: &QueuedJob, _success: bool) {} diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 13b345e36e..af34540add 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -7,13 +7,23 @@ */ use const_format::concatcp; + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; +use tracing::Event; use tracing_appender::non_blocking::{NonBlockingBuilder, WorkerGuard}; +use tracing_subscriber::layer::Context; use tracing_subscriber::{ + filter::Targets, fmt::{format, Layer}, prelude::*, EnvFilter, }; +use crate::utils::Mode; + fn json_layer() -> Layer> { tracing_subscriber::fmt::layer() .json() @@ -34,7 +44,10 @@ pub const LOGS_SERVICE: &str = "logs/services/"; pub const TMP_WINDMILL_LOGS_SERVICE: &str = concatcp!("/tmp/windmill/", LOGS_SERVICE); -pub fn initialize_tracing(hostname: &str) -> WorkerGuard { +pub fn initialize_tracing( + hostname: &str, + mode: &Mode, +) -> (WorkerGuard, crate::otel_ee::OtelProvider) { let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()); if std::env::var("RUST_LOG").is_ok_and(|x| x == "debug" || x == "info") { @@ -44,7 +57,17 @@ pub fn initialize_tracing(hostname: &str) -> WorkerGuard { ) } - let env_filter = EnvFilter::from_default_env(); + let meter_provider = crate::otel_ee::init_meter_provider(mode); + + #[cfg(all(feature = "otel", feature = "enterprise"))] + let opentelemetry = crate::otel_ee::init_otlp_tracer(mode) + .map(|x| tracing_opentelemetry::layer().with_tracer(x)); + + #[cfg(not(all(feature = "otel", feature = "enterprise")))] + let opentelemetry: Option = None; + + let logs_bridge = crate::otel_ee::init_logs_bridge(&mode); + use tracing_appender::rolling::{RollingFileAppender, Rotation}; let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname); @@ -61,39 +84,59 @@ pub fn initialize_tracing(hostname: &str) -> WorkerGuard { .finish(file_appender); let stdout_and_log_file_writer = std::io::stdout.and(log_file_writer); - let ts_base = tracing_subscriber::registry().with(env_filter); + // let job_logs_filter = tracing_subscriber::filter::Targets::new() + // .with_target("windmill:job_log", tracing::Level::TRACE); - #[cfg(feature = "loki")] - let ts_base = { - let (layer, task) = tracing_loki::builder() - .build_url(reqwest::Url::parse("http://127.0.0.1:3100").unwrap()) - .expect("build loki url"); - tokio::spawn(task); - ts_base.with(layer) - }; + let env_filter = EnvFilter::builder() + .with_default_directive(tracing::level_filters::LevelFilter::ERROR.into()) + .from_env_lossy(); + + let ts_base = tracing_subscriber::registry().with(env_filter); match *JSON_FMT { true => ts_base + .with(logs_bridge) + .with(opentelemetry) + // .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap())) .with( json_layer() .with_writer(stdout_and_log_file_writer) - .flatten_event(true), + .flatten_event(true) + .with_filter( + Targets::new() + .with_target( + "windmill:job_log", + tracing::level_filters::LevelFilter::OFF, + ) + .with_default(tracing::level_filters::LevelFilter::INFO), + ), ) .with(CountingLayer::new()) .init(), false => ts_base + .with(logs_bridge) + .with(opentelemetry) + // .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap())) .with( compact_layer() .with_writer(stdout_and_log_file_writer) .with_ansi(style.to_lowercase() != "never") .with_file(true) .with_line_number(true) - .with_target(false), + .with_target(false) + .with_filter( + Targets::new() + .with_target( + "windmill:job_log", + tracing::level_filters::LevelFilter::OFF, + ) + .with_default(tracing::level_filters::LevelFilter::INFO), + ), ) .with(CountingLayer::new()) .init(), } - _guard + (_guard, meter_provider) } #[cfg(feature = "flamegraph")] @@ -112,13 +155,6 @@ pub fn setup_flamegraph() -> impl Drop { _guard } -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; -use tracing::Event; -use tracing_subscriber::layer::Context; - lazy_static::lazy_static! { pub static ref LOG_COUNTING_BY_MIN: Arc>> = Arc::new(RwLock::new(HashMap::new())); } @@ -144,22 +180,6 @@ impl CountingLayer { } } -// impl CountingLayer { -// pub fn new() -> Self { -// CountingLayer { counter: Arc::new(Mutex::new(LogCounter::new())) } -// } - -// pub fn get_counts(&self) -> (usize, usize) { -// let counter = self.counter.lock().unwrap(); -// (counter.non_error_count, counter.error_count) -// } - -// pub fn reset_counts(&self) { -// let mut counter = self.counter.lock().unwrap(); -// counter.reset(); -// } -// } - pub const LOG_TIMESTAMP_FMT: &str = "%Y-%m-%d-%H-%M"; impl tracing_subscriber::Layer for CountingLayer diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index 7ac744a885..8f94de918a 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -43,4 +43,5 @@ bigdecimal.workspace = true axum.workspace = true serde_urlencoded.workspace = true regex.workspace = true -backon.workspace = true \ No newline at end of file +backon.workspace = true +opentelemetry.workspace = true diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index c26a92872f..ef37834400 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -32,7 +32,6 @@ use sqlx::{types::Json, FromRow, Pool, Postgres, Transaction}; #[cfg(feature = "benchmark")] use std::time::Instant; use tokio::{sync::RwLock, time::sleep}; -use tracing::{instrument, Instrument}; use ulid::Ulid; use uuid::Uuid; use windmill_audit::audit_ee::{audit_log, AuditAuthor}; @@ -454,7 +453,6 @@ where } } -#[instrument(level = "trace", skip_all)] pub async fn add_completed_job_error( db: &Pool, queued_job: &QueuedJob, @@ -510,7 +508,6 @@ lazy_static::lazy_static! { pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok(); } -#[instrument(level = "trace", skip_all, name = "add_completed_job")] pub async fn add_completed_job( db: &Pool, queued_job: &QueuedJob, @@ -643,9 +640,7 @@ pub async fn add_completed_job( .fetch_one(&mut *tx) .await .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e:#}")))?; - // tracing::error!("2 {:?}", start.elapsed()); - // add_time!(bench, "add_completed_job query END"); if !queued_job.is_flow_step { if _duration > 500 @@ -1259,7 +1254,6 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> Ok(()) } -#[instrument(level = "trace", skip_all)] pub async fn handle_maybe_scheduled_job<'c>( db: &Pool, job: &QueuedJob, @@ -2429,7 +2423,6 @@ async fn extract_result_from_job_result( } } -#[instrument(level = "trace", skip_all)] pub async fn delete_job<'c>( mut tx: Transaction<'c, Postgres>, w_id: &str, @@ -4039,7 +4032,6 @@ pub async fn push<'c, 'd>( script_path.as_ref().map(|x| x.as_str()), Some(hm), ) - .instrument(tracing::info_span!("job_run", email = &email)) .await?; } diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index cb64757a7f..22640e577a 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -19,6 +19,7 @@ flow_testing = [] cloud = [] sqlx = [] deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", "dep:deno_ast", "dep:deno_tls"] +otel = ["windmill-common/otel", "dep:opentelemetry"] [dependencies] windmill-queue.workspace = true @@ -92,6 +93,9 @@ yaml-rust.workspace = true swc_ecma_parser.workspace = true backon.workspace = true +opentelemetry = { workspace = true, optional = true } + + [build-dependencies] deno_fetch = { workspace = true, optional = true } deno_webidl = { workspace = true, optional = true } diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 31d8ecc4dd..f36954819d 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -866,11 +866,11 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error { #[cfg(windows)] let err_msg = "program not found"; - if err - .to_string() - .contains(&err_msg) - { - return Error::InternalErr(format!("Executable {executable} not found on worker. PATH: {}", *PATH_ENV)); + if err.to_string().contains(&err_msg) { + return Error::InternalErr(format!( + "Executable {executable} not found on worker. PATH: {}", + *PATH_ENV + )); } return err; } diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 4d09c36abc..cb35070a40 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -87,7 +87,7 @@ async fn kill_process_tree(pid: Option) -> Result<(), String> { /// - update the `last_line` and `logs` strings with the program output /// - update "queue"."last_ping" every five seconds /// - kill process if we exceed timeout or "queue"."canceled" is set -#[tracing::instrument(level = "trace", skip_all)] +#[tracing::instrument(name="run_subprocess", level = "info", skip_all, fields(otel.name = %child_name))] pub async fn handle_child( job_id: &Uuid, db: &Pool, @@ -435,8 +435,6 @@ pub async fn handle_child( } } - - async fn get_mem_peak(pid: Option, nsjail: bool) -> i32 { if pid.is_none() { return -1; @@ -692,7 +690,10 @@ fn child_joined_output_stream( let stdout = BufReader::new(stdout).lines(); let stderr = BufReader::new(stderr).lines(); - stream::select(lines_to_stream(stderr, true), lines_to_stream(stdout, false)) + stream::select( + lines_to_stream(stderr, true), + lines_to_stream(stdout, false), + ) } pub fn lines_to_stream( @@ -702,14 +703,10 @@ pub fn lines_to_stream( stream::poll_fn(move |cx| { std::pin::Pin::new(&mut lines) .poll_next_line(cx) - .map(|result| { - process_streaming_log_lines(result, stderr) - }) + .map(|result| process_streaming_log_lines(result, stderr)) }) } - - pub fn process_status(status: ExitStatus) -> error::Result<()> { if status.success() { Ok(()) diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 25bfade2e8..a897459224 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -1,3 +1,6 @@ +#[cfg(feature = "otel")] +use opentelemetry::trace::FutureExt; + use serde::Serialize; use sqlx::{types::Json, Pool, Postgres}; use std::{ @@ -7,6 +10,9 @@ use std::{ Arc, }, }; +use tracing::{field, Instrument}; +#[cfg(not(feature = "otel"))] +use windmill_common::otel_ee::FutureExt; use uuid::Uuid; @@ -85,7 +91,49 @@ pub fn start_background_processor( JobKind::Dependencies | JobKind::FlowDependencies ); - handle_receive_completed_job( + let success = jc.success; + + let span = tracing::span!( + tracing::Level::INFO, + "job_postprocessing", + job_id = %jc.job.id, root_job = field::Empty, workspace_id = %jc.job.workspace_id, worker = %worker_name,tag = %jc.job.tag, + // hostname = %hostname, + language = field::Empty, + script_path = field::Empty, + flow_step_id = field::Empty, + parent_job = field::Empty, + otel.name = field::Empty + ); + let rj = if let Some(root_job) = jc.job.root_job { + root_job + } else { + jc.job.id + }; + windmill_common::otel_ee::set_span_parent(&span, &rj); + + if let Some(lg) = jc.job.language.as_ref() { + span.record("language", lg.as_str()); + } + if let Some(step_id) = jc.job.flow_step_id.as_ref() { + span.record( + "otel.name", + format!("job_postprocessing {}", step_id).as_str(), + ); + span.record("flow_step_id", step_id.as_str()); + } else { + span.record("otel.name", "job postprocessing"); + } + if let Some(parent_job) = jc.job.parent_job.as_ref() { + span.record("parent_job", parent_job.to_string().as_str()); + } + if let Some(script_path) = jc.job.script_path.as_ref() { + span.record("script_path", script_path.as_str()); + } + if let Some(root_job) = jc.job.root_job.as_ref() { + span.record("root_job", root_job.to_string().as_str()); + } + + let root_job = handle_receive_completed_job( jc, &base_internal_url, &db, @@ -96,8 +144,14 @@ pub fn start_background_processor( #[cfg(feature = "benchmark")] &mut bench, ) + .instrument(span) .await; + if let Some(root_job) = root_job { + windmill_common::otel_ee::add_root_flow_job_to_otlp(&root_job, success); + tracing::error!(job_id = %root_job.id, parent_job = ?root_job.parent_job, "ADDDED root job completed"); + } + if is_init_script_and_failure { tracing::error!("init script errored, exiting"); killpill_tx.send(()).unwrap_or_default(); @@ -199,7 +253,11 @@ async fn send_job_completed( token, duration, }; - job_completed_tx.send(jc).await.expect("send job completed") + job_completed_tx + .send(jc) + .with_context(windmill_common::otel_ee::otel_ctx()) + .await + .expect("send job completed") } pub async fn process_result( @@ -271,6 +329,7 @@ pub async fn process_result( token, duration, ) + .with_context(windmill_common::otel_ee::otel_ctx()) .await; Ok(true) } @@ -315,6 +374,7 @@ pub async fn process_result( token, duration, ) + .with_context(windmill_common::otel_ee::otel_ctx()) .await; Ok(false) } @@ -330,7 +390,7 @@ pub async fn handle_receive_completed_job( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) { +) -> Option> { let token = jc.token.clone(); let workspace = jc.job.workspace_id.clone(); let client = AuthedClient { @@ -342,7 +402,7 @@ pub async fn handle_receive_completed_job( let job = jc.job.clone(); let mem_peak = jc.mem_peak.clone(); let canceled_by = jc.canceled_by.clone(); - if let Err(err) = process_completed_job( + match process_completed_job( jc, &client, db, @@ -355,26 +415,29 @@ pub async fn handle_receive_completed_job( ) .await { - handle_job_error( - db, - &client, - job.as_ref(), - mem_peak, - canceled_by, - err, - false, - same_worker_tx.clone(), - &worker_dir, - worker_name, - job_completed_tx, - #[cfg(feature = "benchmark")] - bench, - ) - .await; + Err(err) => { + handle_job_error( + db, + &client, + job.as_ref(), + mem_peak, + canceled_by, + err, + false, + same_worker_tx.clone(), + &worker_dir, + worker_name, + job_completed_tx, + #[cfg(feature = "benchmark")] + bench, + ) + .await; + None + } + Ok(r) => r, } } -#[tracing::instrument(name = "completed_job", level = "info", skip_all, fields(job_id = %job.id))] pub async fn process_completed_job( JobCompleted { job, result, mem_peak, success, cached_res_path, canceled_by, duration, .. }: JobCompleted, client: &AuthedClient, @@ -384,7 +447,7 @@ pub async fn process_completed_job( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) -> windmill_common::error::Result<()> { +) -> windmill_common::error::Result>> { if success { // println!("bef completed job{:?}", SystemTime::now()); if let Some(cached_path) = cached_res_path { @@ -414,8 +477,8 @@ pub async fn process_completed_job( if is_flow_step { if let Some(parent_job) = parent_job { - tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)"); - update_flow_status_after_job_completion( + // tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)"); + let r = update_flow_status_after_job_completion( db, client, parent_job, @@ -434,9 +497,10 @@ pub async fn process_completed_job( ) .warn_after_seconds(10) .await?; + add_time!(bench, "updated flow status END"); + return Ok(r); } } - add_time!(bench, "updated flow status END"); } else { let result = add_completed_job_error( db, @@ -454,7 +518,7 @@ pub async fn process_completed_job( if job.is_flow_step { if let Some(parent_job) = job.parent_job { tracing::error!(parent_flow = %parent_job, subflow = %job.id, "process completed job error, updating flow status"); - update_flow_status_after_job_completion( + let r = update_flow_status_after_job_completion( db, client, parent_job, @@ -473,10 +537,11 @@ pub async fn process_completed_job( ) .warn_after_seconds(10) .await?; + return Ok(r); } } } - Ok(()) + return Ok(None); } #[tracing::instrument(name = "job_error", level = "info", skip_all, fields(job_id = %job.id))] diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 238fd6a951..f179396123 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -6,6 +6,10 @@ * LICENSE-AGPL for a copy of the license. */ +// #[cfg(feature = "otel")] +// use opentelemetry::{global, KeyValue}; + + use windmill_common::{ apps::AppScriptId, auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET}, @@ -25,7 +29,7 @@ use const_format::concatcp; #[cfg(feature = "prometheus")] use prometheus::IntCounter; -use tracing::Instrument; +use tracing::{field, Instrument}; #[cfg(feature = "prometheus")] use windmill_common::METRICS_DEBUG_ENABLED; #[cfg(feature = "prometheus")] @@ -88,30 +92,12 @@ use tokio::{ use rand::Rng; use crate::{ - ansible_executor::handle_ansible_job, - bash_executor::{handle_bash_job, handle_powershell_job}, - bun_executor::handle_bun_job, - common::{ + ansible_executor::handle_ansible_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{ build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, update_worker_ping_for_failed_init_script, OccupancyMetrics, - }, - deno_executor::handle_deno_job, - go_executor::handle_go_job, - graphql_executor::do_graphql, - handle_child::SLOW_LOGS, - handle_job_error, - job_logger::NO_LOGS_AT_ALL, - js_eval::{eval_fetch_timeout, transpile_ts}, - mysql_executor::do_mysql, - pg_executor::do_postgresql, - php_executor::handle_php_job, - python_executor::handle_python_job, - result_processor::{process_result, start_background_processor}, - rust_executor::handle_rust_job, - worker_flow::{handle_flow, update_flow_status_in_progress, Step}, - worker_lockfiles::{ + }, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, handle_child::SLOW_LOGS, handle_job_error, job_logger::NO_LOGS_AT_ALL, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{process_result, start_background_processor}, rust_executor::handle_rust_job, worker_flow::{handle_flow, update_flow_status_in_progress, Step}, worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, - }, + } }; use backon::ConstantBuilder; @@ -720,7 +706,10 @@ fn add_outstanding_wait_time( }.in_current_span()); } -#[tracing::instrument(name = "worker", level = "info", skip_all, fields(worker = %worker_name, hostname = %hostname))] +// struct WorkerMtrics { +// job_ +// } + pub async fn run_worker( db: &Pool, hostname: &str, @@ -736,6 +725,7 @@ pub async fn run_worker( #[cfg(not(feature = "enterprise"))] if !*DISABLE_NSJAIL { tracing::warn!( + worker = %worker_name, hostname = %hostname, "NSJAIL to sandbox process in untrusted environments is an enterprise feature but allowed to be used for testing purposes" ); } @@ -743,10 +733,10 @@ pub async fn run_worker( let start_time = Instant::now(); let worker_dir = format!("{TMP_DIR}/{worker_name}"); - tracing::debug!(worker_dir = %worker_dir, "Creating worker dir"); + tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir"); if let Some(ref netrc) = *NETRC { - tracing::info!("Writing netrc at {}/.netrc", HOME_ENV.as_str()); + tracing::info!(worker = %worker_name, hostname = %hostname, "Writing netrc at {}/.netrc", HOME_ENV.as_str()); write_file(&HOME_ENV, ".netrc", netrc).expect("could not write netrc"); } @@ -961,6 +951,16 @@ pub async fn run_worker( None }; + + // let worker_resource = &[ + // KeyValue::new("hostname", hostname.to_string()), + // KeyValue::new("worker", worker_name.to_string()), + // ]; + // // Create a meter from the above MeterProvider. + // let meter = global::meter("windmill"); + // let counter = meter.u64_counter("jobs.execution").build(); + + let mut occupancy_metrics = OccupancyMetrics::new(start_time); let mut jobs_executed = 0; @@ -1016,6 +1016,7 @@ pub async fn run_worker( IS_READY.store(true, Ordering::Relaxed); tracing::info!( + worker = %worker_name, hostname = %hostname, "listening for jobs, WORKER_GROUP: {}, config: {:?}", *WORKER_GROUP, WORKER_CONFIG.read().await @@ -1051,7 +1052,7 @@ pub async fn run_worker( if i_worker == 1 { if let Err(e) = queue_init_bash_maybe(db, same_worker_tx.clone(), &worker_name).await { killpill_tx.send(()).unwrap_or_default(); - tracing::error!("Error queuing init bash script for worker {worker_name}: {e:#}"); + tracing::error!(worker = %worker_name, hostname = %hostname, "Error queuing init bash script for worker {worker_name}: {e:#}"); return; } } @@ -1084,7 +1085,7 @@ pub async fn run_worker( #[cfg(feature = "enterprise")] { if let Ok(_) = killpill_rx.try_recv() { - tracing::info!("killpill received on worker waiting for valid key"); + tracing::info!(worker = %worker_name, hostname = %hostname, "killpill received on worker waiting for valid key"); job_completed_tx .0 .send(SendResult::Kill) @@ -1096,6 +1097,7 @@ pub async fn run_worker( if !valid_key { tracing::error!( + worker = %worker_name, hostname = %hostname, "Invalid license key, workers require a valid license key, sleeping for 30s waiting for valid key to be set" ); tokio::time::sleep(Duration::from_secs(10)).await; @@ -1109,7 +1111,7 @@ pub async fn run_worker( #[cfg(feature = "prometheus")] if let Some(wk) = worker_busy.as_ref() { wk.set(0); - tracing::debug!("set worker busy to 0"); + tracing::debug!(worker = %worker_name, hostname = %hostname, "set worker busy to 0"); } occupancy_metrics.running_job_started_at = None; @@ -1121,7 +1123,7 @@ pub async fn run_worker( .try_into() .unwrap(), ); - tracing::debug!("set uptime metric"); + tracing::debug!(worker = %worker_name, hostname = %hostname, "set uptime metric"); } if last_ping.elapsed().as_secs() > NUM_SECS_PING { @@ -1165,15 +1167,19 @@ pub async fn run_worker( ) .notify(|err, dur| { tracing::error!( + worker = %worker_name, hostname = %hostname, "retrying updating worker ping in {dur:#?}, err: {err:#?}" ); }) .sleep(tokio::time::sleep) .await { - tracing::error!("failed to update worker ping, exiting: {}", e); + tracing::error!( + worker = %worker_name, hostname = %hostname, + "failed to update worker ping, exiting: {}", e); killpill_tx.send(()).unwrap_or_default(); } tracing::info!( + worker = %worker_name, hostname = %hostname, "ping update, memory: container={}MB, windmill={}MB", memory_usage.unwrap_or_default() / (1024 * 1024), wm_memory_usage.unwrap_or_default() / (1024 * 1024) @@ -1185,16 +1191,18 @@ pub async fn run_worker( if (jobs_executed as u32 + vacuum_shift) % VACUUM_PERIOD == 0 { let db2 = db.clone(); let current_span = tracing::Span::current(); + let worker_name = worker_name.clone(); + let hostname = hostname.to_string(); tokio::task::spawn( (async move { - tracing::info!("vacuuming queue and completed_job"); + tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue"); if let Err(e) = sqlx::query!("VACUUM (skip_locked) queue") .execute(&db2) .await { - tracing::error!("failed to vacuum queue: {}", e); + tracing::error!(worker = %worker_name, hostname = %hostname, "failed to vacuum queue: {}", e); } - tracing::info!("vacuumed queue and completed_job"); + tracing::info!(worker = %worker_name, hostname = %hostname, "vacuumed queue"); }) .instrument(current_span), ); @@ -1230,6 +1238,7 @@ pub async fn run_worker( if let Ok(same_worker_job) = same_worker_rx.try_recv() { same_worker_queue_size.fetch_sub(1, Ordering::SeqCst); tracing::debug!( + worker = %worker_name, hostname = %hostname, "received {} from same worker channel", same_worker_job.job_id ); @@ -1242,6 +1251,7 @@ pub async fn run_worker( .map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string())); if r.is_err() && !same_worker_job.recoverable { tracing::error!( + worker = %worker_name, hostname = %hostname, "failed to fetch same_worker job on a non recoverable job, exiting" ); job_completed_tx @@ -1255,7 +1265,7 @@ pub async fn run_worker( } } else if let Ok(_) = killpill_rx.try_recv() { if !killed_but_draining_same_worker_jobs { - tracing::info!("received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker); + tracing::info!(worker = %worker_name, hostname = %hostname, "received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker); killed_but_draining_same_worker_jobs = true; job_completed_tx .0 @@ -1266,10 +1276,10 @@ pub async fn run_worker( continue; } else if killed_but_draining_same_worker_jobs { if job_completed_processor_is_done.load(Ordering::SeqCst) { - tracing::info!("all running jobs have completed and all completed jobs have been fully processed, exiting"); + tracing::info!(worker = %worker_name, hostname = %hostname, "all running jobs have completed and all completed jobs have been fully processed, exiting"); break; } else { - tracing::info!("there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting"); + tracing::info!(worker = %worker_name, hostname = %hostname, "there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting"); tokio::time::sleep(Duration::from_millis(200)).await; continue; } @@ -1294,7 +1304,7 @@ pub async fn run_worker( if !agent_mode && duration_pull_s > 0.5 { let empty = job.as_ref().is_ok_and(|x| x.0.is_none()); - tracing::warn!("pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}"); + tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}"); #[cfg(feature = "prometheus")] if empty { if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() { @@ -1305,7 +1315,7 @@ pub async fn run_worker( } } else if !agent_mode && duration_pull_s > 0.1 { let empty = job.as_ref().is_ok_and(|x| x.0.is_none()); - tracing::warn!("pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}"); + tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}"); #[cfg(feature = "prometheus")] if empty { if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() { @@ -1359,7 +1369,7 @@ pub async fn run_worker( last_executed_job = None; jobs_executed += 1; - tracing::debug!("started handling of job {}", job.id); + tracing::debug!(worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id); if matches!(job.job_kind, JobKind::Script | JobKind::Preview) { if !dedicated_workers.is_empty() { @@ -1424,6 +1434,11 @@ pub async fn run_worker( ) .await; + // counter.add( + // 1, + // worker_resource + // ); + #[cfg(feature = "prometheus")] let _timer = register_metric( &WORKER_EXECUTION_DURATION, @@ -1511,6 +1526,40 @@ pub async fn run_worker( let PulledJob { job, raw_code, raw_lock, raw_flow } = job; let arc_job = Arc::new(job); add_time!(bench, "handle_queued_job START"); + + + let span = tracing::span!(tracing::Level::INFO, "job", + job_id = %arc_job.id, root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, hostname = %hostname, tag = %arc_job.tag, + language = field::Empty, + script_path = field::Empty, flow_step_id = field::Empty, parent_job = field::Empty, + otel.name = field::Empty); + let rj = if let Some(root_job) = arc_job.root_job { + root_job + } else { + arc_job.id + }; + if let Some(lg) = arc_job.language.as_ref() { + span.record("language", lg.as_str()); + } + if let Some(step_id) = arc_job.flow_step_id.as_ref() { + span.record("otel.name", format!("job {}", step_id).as_str()); + span.record("flow_step_id", step_id.as_str()); + } else { + span.record("otel.name", "job"); + } + if let Some(parent_job) = arc_job.parent_job.as_ref() { + span.record("parent_job", parent_job.to_string().as_str()); + } + if let Some(script_path) = arc_job.script_path.as_ref() { + span.record("script_path", script_path.as_str()); + } + if let Some(root_job) = arc_job.root_job.as_ref() { + span.record("root_job", root_job.to_string().as_str()); + } + + windmill_common::otel_ee::set_span_parent(&span, &rj); + // span.context().span().add_event_with_timestamp("job created".to_string(), arc_job.created_at.into(), vec![]); + match handle_queued_job( arc_job.clone(), raw_code, @@ -1529,6 +1578,7 @@ pub async fn run_worker( #[cfg(feature = "benchmark")] &mut bench, ) + .instrument(span) .await { Err(err) => { @@ -1568,6 +1618,8 @@ pub async fn run_worker( _ => {} } + + #[cfg(feature = "prometheus")] if let Some(duration) = _timer.map(|x| x.stop_and_record()) { register_metric( @@ -1607,7 +1659,7 @@ pub async fn run_worker( if let Some(secs) = *EXIT_AFTER_NO_JOB_FOR_SECS { if let Some(lj) = last_executed_job { if lj.elapsed().as_secs() > secs { - tracing::info!("no job for {} seconds, exiting", secs); + tracing::info!(worker = %worker_name, hostname = %hostname, "no job for {} seconds, exiting", secs); break; } } else { @@ -1638,12 +1690,12 @@ pub async fn run_worker( }); } Err(err) => { - tracing::error!("Failed to pull jobs: {}", err); + tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err); } }; } - tracing::info!("worker {} exiting", worker_name); + tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exiting", worker_name); #[cfg(feature = "benchmark")] { @@ -1658,22 +1710,24 @@ pub async fn run_worker( if has_dedicated_workers { for handle in dedicated_handles { if let Err(e) = handle.await { - tracing::error!("error in dedicated worker waiting for it to end: {:?}", e) + tracing::error!(worker = %worker_name, hostname = %hostname, "error in dedicated worker waiting for it to end: {:?}", e) } } - tracing::info!("all dedicated workers have exited"); + tracing::info!(worker = %worker_name, hostname = %hostname, "all dedicated workers have exited"); } drop(job_completed_tx); - tracing::info!("waiting for job_completed_processor to finish processing remaining jobs"); + tracing::info!(worker = %worker_name, hostname = %hostname, "waiting for job_completed_processor to finish processing remaining jobs"); if let Err(e) = send_result.await { tracing::error!("error in awaiting send_result process: {e:?}") } - tracing::info!("worker {} exited", worker_name); - tracing::info!("number of jobs executed: {}", jobs_executed); + tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name); + tracing::info!(worker = %worker_name, hostname = %hostname, "number of jobs executed: {}", jobs_executed); } + + async fn queue_init_bash_maybe<'c>( db: &Pool, same_worker_tx: SameWorkerSender, @@ -1798,7 +1852,6 @@ pub struct PreviousResult<'a> { pub previous_result: Option<&'a RawValue>, } -#[tracing::instrument(name = "job", level = "info", skip_all, fields(job_id = %job.id))] async fn handle_queued_job( job: Arc, raw_code: Option, @@ -1816,6 +1869,9 @@ async fn handle_queued_job( occupancy_metrics: &mut OccupancyMetrics, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { + // Extract the active span from the context + + if job.canceled { return Err(Error::JsonErr(canceled_job_to_result(&job))); } @@ -2159,6 +2215,7 @@ async fn handle_queued_job( } } + pub fn build_envs( envs: Option>, ) -> windmill_common::error::Result> { diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 74949d0bcf..b5770166c5 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -77,45 +77,34 @@ pub async fn update_flow_status_after_job_completion( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) -> error::Result<()> { +) -> error::Result>> { // this is manual tailrecursion because async_recursion blows up the stack - // todo!(); potentially_crash_for_testing(); - let mut rec = update_flow_status_after_job_completion_internal( - db, - client, + let mut rec = RecUpdateFlowStatusAfterJobCompletion { flow, - job_id_for_status, - w_id, + job_id_for_status: job_id_for_status.clone(), success, result, - unrecoverable, - same_worker_tx.clone(), - worker_dir, stop_early_override, - false, - worker_name, - job_completed_tx.clone(), - #[cfg(feature = "benchmark")] - bench, - ) - .await?; - while let Some(nrec) = rec { + skip_error_handler: false, + }; + let mut unrecoverable = unrecoverable; + loop { potentially_crash_for_testing(); - rec = match update_flow_status_after_job_completion_internal( + let nrec = match update_flow_status_after_job_completion_internal( db, client, - nrec.flow, - &nrec.job_id_for_status, + rec.flow, + &rec.job_id_for_status, w_id, - nrec.success, - nrec.result, - false, + rec.success, + rec.result, + unrecoverable, same_worker_tx.clone(), worker_dir, - nrec.stop_early_override, - nrec.skip_error_handler, + rec.stop_early_override, + rec.skip_error_handler, worker_name, job_completed_tx.clone(), #[cfg(feature = "benchmark")] @@ -125,12 +114,12 @@ pub async fn update_flow_status_after_job_completion( { Ok(j) => j, Err(e) => { - tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", nrec.flow,&nrec.job_id_for_status); + tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", rec.flow, &rec.job_id_for_status); update_flow_status_after_job_completion_internal( db, client, - nrec.flow, - &nrec.job_id_for_status, + rec.flow, + &rec.job_id_for_status, w_id, false, Arc::new(to_raw_value(&Json(&WrappedError { @@ -139,8 +128,8 @@ pub async fn update_flow_status_after_job_completion( true, same_worker_tx.clone(), worker_dir, - nrec.stop_early_override, - nrec.skip_error_handler, + rec.stop_early_override, + rec.skip_error_handler, worker_name, job_completed_tx.clone(), #[cfg(feature = "benchmark")] @@ -148,9 +137,33 @@ pub async fn update_flow_status_after_job_completion( ) .await? } + }; + unrecoverable = false; + match nrec { + UpdateFlowStatusAfterJobCompletion::Done(job) => { + add_time!(bench, "update flow status internal END"); + return Ok(Some(job)); + } + UpdateFlowStatusAfterJobCompletion::Rec(nrec) => { + rec = nrec; + }, + UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch => { + add_time!(bench, "update flow status internal END"); + return Ok(None); + }, + UpdateFlowStatusAfterJobCompletion::NotDone => { + add_time!(bench, "update flow status internal END"); + return Ok(None); + } } } - Ok(()) +} + +pub enum UpdateFlowStatusAfterJobCompletion { + Rec(RecUpdateFlowStatusAfterJobCompletion), + Done(Arc), + NotDone, + NonLastParallelBranch, } pub struct RecUpdateFlowStatusAfterJobCompletion { flow: uuid::Uuid, @@ -188,7 +201,7 @@ pub async fn update_flow_status_after_job_completion_internal( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) -> error::Result> { +) -> error::Result { add_time!(bench, "update flow status internal START"); let ( should_continue_flow, @@ -603,7 +616,7 @@ pub async fn update_flow_status_after_job_completion_internal( ); } add_time!(bench, "non final parallel flow finished"); - return Ok(None); + return Ok(UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch); } } FlowStatusModule::InProgress { @@ -1148,7 +1161,7 @@ pub async fn update_flow_status_after_job_completion_internal( if let Some(parent_job) = flow_job.parent_job { tracing::info!(subflow_id = %flow_job.id, parent_id = %parent_job, "subflow is finished, updating parent flow status"); - return Ok(Some(RecUpdateFlowStatusAfterJobCompletion { + return Ok(UpdateFlowStatusAfterJobCompletion::Rec(RecUpdateFlowStatusAfterJobCompletion { flow: parent_job, job_id_for_status: flow, success: success && !is_failure_step, @@ -1162,9 +1175,9 @@ pub async fn update_flow_status_after_job_completion_internal( })); } } - Ok(None) + Ok(UpdateFlowStatusAfterJobCompletion::Done(flow_job)) } else { - Ok(None) + Ok(UpdateFlowStatusAfterJobCompletion::NotDone) } } diff --git a/frontend/src/lib/components/AuthSettings.svelte b/frontend/src/lib/components/AuthSettings.svelte new file mode 100644 index 0000000000..56f6d625de --- /dev/null +++ b/frontend/src/lib/components/AuthSettings.svelte @@ -0,0 +1,255 @@ + + +
+ + SSO + OAuth + SCIM/SAML + +
+ +
+ {#if tab === 'sso'} + {#if !$enterpriseLicense || $enterpriseLicense.endsWith('_pro')} + + Without EE, the number of SSO users is limited to 10. SCIM/SAML is available on EE + + {/if} + +
+
+ When at least one of the below options is set, users will be able to login to Windmill via + their third-party account. +
To test SSO, the recommended workflow is to to save the settings and try to login in + an incognito window. + Learn more +
+
+ + + + + + + + + + + + {#each Object.keys(oauths) as k} + {#if !['authelia', 'authentik', 'google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'keycloak', 'slack', 'kanidm', 'zitadel'].includes(k) && 'login_config' in oauths[k]} + {#if oauths[k]} +
+
+ + + { + delete oauths[k] + oauths = { ...oauths } + }} + /> +
+
+ + + + {#if !windmillBuiltins.includes(k) && k != 'slack'} + + {/if} +
+
+ {/if} + {/if} + {/each} +
+
+ + +
+
+ +
+ {:else if tab === 'oauth'} +
+ When one of the below options is set, you will be able to create a specific resource + containing a token automatically generated by the third-party provider. +
+ To test it after setting an oauth client, go to the Resources menu and create a new one of the + type of your oauth client (i.e. a 'github' resource if you set Github OAuth). +
Learn more +
+
+ +
+ + {#each Object.keys(oauths) as k} + {#if oauths[k] && !('login_config' in oauths[k])} + {#if !['slack'].includes(k) && oauths[k]} +
+
+ + + { + delete oauths[k] + oauths = { ...oauths } + }} + /> +
+
+ + + {#if !windmillBuiltins.includes(k) && k != 'slack'} + + {/if} + {#if k == 'snowflake_oauth'} + + {/if} +
+
+ {/if} + {/if} + {/each} + +
+ + {#if oauth_name == 'custom'} + + {:else} + + {/if} + +
+ {:else if tab == 'scim'} + + {/if} +
diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte new file mode 100644 index 0000000000..1bac352d5b --- /dev/null +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -0,0 +1,783 @@ + + +{#if (!setting.cloudonly || isCloudHosted()) && showSetting(setting.key, $values) && !(setting.hiddenIfNull && $values[setting.key] == null)} + {#if setting.ee_only != undefined && !$enterpriseLicense} +
+ + EE only {#if setting.ee_only != ''}{setting.ee_only}{/if} +
+ {/if} + +