From 10b6b1dc04cb2b2d4ec5ceb26a5dbd2ac7bd1394 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 14 Nov 2024 11:03:26 +0100 Subject: [PATCH 01/31] fix: add queue_couts api --- backend/windmill-api/openapi.yaml | 17 +++++++++++++++++ backend/windmill-api/src/workers.rs | 10 ++++++++++ 2 files changed, 27 insertions(+) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index a85619dde7..97d98d3d6b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -8188,6 +8188,23 @@ paths: - id - values + /workers/queue_counts: + get: + summary: get counts of jobs waiting for an executor per tag + operationId: getCountsOfJobsWaitingPerTag + tags: + - worker + responses: + "200": + description: queue counts + content: + application/json: + schema: + type: object + additionalProperties: + type: integer + + /configs/list_worker_groups: get: summary: list worker groups diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index 2f8439bf24..6eeeb17bac 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -36,6 +36,7 @@ pub fn global_service() -> Router { ) .route("/get_default_tags", get(get_default_tags)) .route("/queue_metrics", get(get_queue_metrics)) + .route("/queue_counts", get(get_queue_counts)) } #[derive(FromRow, Serialize, Deserialize)] @@ -176,3 +177,12 @@ async fn get_queue_metrics( Ok(Json(queue_metrics)) } + +async fn get_queue_counts( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + require_super_admin(&db, &authed.email).await?; + let queue_counts = windmill_common::queue::get_queue_counts(&db).await; + Ok(Json(queue_counts)) +} \ No newline at end of file From 4fdca87de9153e04b2b2c21a03165eb03ffc074f Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Thu, 14 Nov 2024 12:23:27 +0100 Subject: [PATCH 02/31] nit: cleanup `raw_flow` usage (#4707) * nit: cleanup `raw_flow` usage * nit: refactor two queries into one --- ...76f06c6077c619c0ab1ac4edf633b22df98c3.json | 23 ++++ ...e4be4506bfd238c88f1b1c3ddf39b29071446.json | 29 +++++ ...610c021a0a5b63ffe5afd7878385c8af6bc6c.json | 24 ++++ ...8359f70eade6654b5a884915f07f3ef3fe15e.json | 23 ++++ backend/windmill-worker/src/worker.rs | 2 + backend/windmill-worker/src/worker_flow.rs | 110 +++++++----------- 6 files changed, 145 insertions(+), 66 deletions(-) create mode 100644 backend/.sqlx/query-0f04f42a596b6759a84ba7d26ea76f06c6077c619c0ab1ac4edf633b22df98c3.json create mode 100644 backend/.sqlx/query-1e7ce0c140410ae799f9c0c5772e4be4506bfd238c88f1b1c3ddf39b29071446.json create mode 100644 backend/.sqlx/query-cd40b9c552d76664a552457c4a9610c021a0a5b63ffe5afd7878385c8af6bc6c.json create mode 100644 backend/.sqlx/query-de1abe57b6aa61155f747a3bcb98359f70eade6654b5a884915f07f3ef3fe15e.json diff --git a/backend/.sqlx/query-0f04f42a596b6759a84ba7d26ea76f06c6077c619c0ab1ac4edf633b22df98c3.json b/backend/.sqlx/query-0f04f42a596b6759a84ba7d26ea76f06c6077c619c0ab1ac4edf633b22df98c3.json new file mode 100644 index 0000000000..bf47f131c4 --- /dev/null +++ b/backend/.sqlx/query-0f04f42a596b6759a84ba7d26ea76f06c6077c619c0ab1ac4edf633b22df98c3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_status AS \"_id!: Json>\" FROM queue WHERE id = $1 AND workspace_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "_id!: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "0f04f42a596b6759a84ba7d26ea76f06c6077c619c0ab1ac4edf633b22df98c3" +} diff --git a/backend/.sqlx/query-1e7ce0c140410ae799f9c0c5772e4be4506bfd238c88f1b1c3ddf39b29071446.json b/backend/.sqlx/query-1e7ce0c140410ae799f9c0c5772e4be4506bfd238c88f1b1c3ddf39b29071446.json new file mode 100644 index 0000000000..066c50aa73 --- /dev/null +++ b/backend/.sqlx/query-1e7ce0c140410ae799f9c0c5772e4be4506bfd238c88f1b1c3ddf39b29071446.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n flow_status AS \"flow_status!: Json>\",\n raw_flow->'modules'->(flow_status->'step')::int AS \"module: Json>\"\n FROM queue WHERE id = $1 AND workspace_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flow_status!: Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "module: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + null + ] + }, + "hash": "1e7ce0c140410ae799f9c0c5772e4be4506bfd238c88f1b1c3ddf39b29071446" +} diff --git a/backend/.sqlx/query-cd40b9c552d76664a552457c4a9610c021a0a5b63ffe5afd7878385c8af6bc6c.json b/backend/.sqlx/query-cd40b9c552d76664a552457c4a9610c021a0a5b63ffe5afd7878385c8af6bc6c.json new file mode 100644 index 0000000000..382caea08a --- /dev/null +++ b/backend/.sqlx/query-cd40b9c552d76664a552457c4a9610c021a0a5b63ffe5afd7878385c8af6bc6c.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT raw_flow->'modules'->($1)::int AS \"_id: Json>\" FROM queue WHERE id = $2 AND workspace_id = $3 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "_id: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "cd40b9c552d76664a552457c4a9610c021a0a5b63ffe5afd7878385c8af6bc6c" +} diff --git a/backend/.sqlx/query-de1abe57b6aa61155f747a3bcb98359f70eade6654b5a884915f07f3ef3fe15e.json b/backend/.sqlx/query-de1abe57b6aa61155f747a3bcb98359f70eade6654b5a884915f07f3ef3fe15e.json new file mode 100644 index 0000000000..3bd7a78a01 --- /dev/null +++ b/backend/.sqlx/query-de1abe57b6aa61155f747a3bcb98359f70eade6654b5a884915f07f3ef3fe15e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT raw_flow->'modules'->($1)::text->'value'->>'type' = 'flow' FROM queue WHERE id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "de1abe57b6aa61155f747a3bcb98359f70eade6654b5a884915f07f3ef3fe15e" +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 1b7dcfa970..28a94d5f92 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -1903,8 +1903,10 @@ async fn handle_queued_job( } }; if job.is_flow() { + let flow = job.parse_raw_flow(); handle_flow( job, + flow, db, &client.get_authed().await, None, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 5421b502e4..0a9484ed88 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -8,7 +8,6 @@ use std::collections::hash_map::DefaultHasher; use std::collections::HashMap; -use std::hash::Hash; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -177,11 +176,6 @@ struct RecoveryObject { recover: Option, } -#[derive(sqlx::FromRow, Deserialize)] -pub struct RowFlowStatus { - pub flow_status: sqlx::types::Json>, - pub current_module: Option>>, -} // #[instrument(level = "trace", skip_all)] pub async fn update_flow_status_after_job_completion_internal< R: rsmq_async::RsmqConnection + Send + Sync + Clone, @@ -207,6 +201,7 @@ pub async fn update_flow_status_after_job_completion_internal< let ( should_continue_flow, flow_job, + flow_value, stop_early, skip_if_stop_early, nresult, @@ -215,35 +210,28 @@ pub async fn update_flow_status_after_job_completion_internal< ) = { // tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}"); - let old_status_json = sqlx::query_as::<_, RowFlowStatus>( - "SELECT flow_status, raw_flow->'modules'->(flow_status->'step')::int as current_module FROM queue WHERE id = $1 AND workspace_id = $2", + let (old_status, current_module) = sqlx::query!( + "SELECT + flow_status AS \"flow_status!: Json>\", + raw_flow->'modules'->(flow_status->'step')::int AS \"module: Json>\" + FROM queue WHERE id = $1 AND workspace_id = $2 LIMIT 1", + flow, w_id ) - .bind(flow) - .bind(w_id) .fetch_one(db) .await - .map_err(|e| { - Error::InternalErr(format!( - "fetching flow status {flow} while reporting {success} {result:?}: {e:#}" - )) - })?; - - let old_status = serde_json::from_str::(old_status_json.flow_status.get()) - .or_else(|e| { - Err(Error::InternalErr(format!( - "requiring status to be parsable as FlowStatus: {e:?}" - ))) - })?; - - let current_module = if let Some(x) = old_status_json.current_module { - Some(serde_json::from_str::(x.0.get()).or_else(|e| { - Err(Error::InternalErr(format!( + .map_err(|e| Error::InternalErr( + format!("fetching flow status {flow} while reporting {success} {result:?}: {e:#}") + )) + .and_then(|record| Ok(( + serde_json::from_str::(record.flow_status.0.get()).map_err(|e| Error::InternalErr( + format!("requiring current module to be parsable as FlowStatus: {e:?}") + ))?, + record.module.map(|json| { + serde_json::from_str::(json.0.get()).map_err(|e| Error::InternalErr(format!( "requiring current module to be parsable as FlowModule: {e:?}" ))) - })?) - } else { - None - }; + }).transpose()?, + )))?; let module_step = Step::from_i32_and_len(old_status.step, old_status.modules.len()); @@ -303,16 +291,15 @@ pub async fn update_flow_status_after_job_completion_internal< let is_flow = if let Some(step) = step { sqlx::query_scalar!( - "SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2", - step as i32, - &flow + "SELECT raw_flow->'modules'->($1)::text->'value'->>'type' = 'flow' FROM queue WHERE id = $2 LIMIT 1", + step as i32, flow ) - .fetch_one(db) - .await - .map_err(|e| { - Error::InternalErr(format!("error during retrieval of step's type: {e:#}")) - })? - .unwrap_or(false) + .fetch_one(db) + .await + .map_err(|e| { + Error::InternalErr(format!("error during retrieval of step's type: {e:#}")) + })? + .unwrap_or(false) } else { false }; @@ -938,10 +925,7 @@ pub async fn update_flow_status_after_job_completion_internal< .unwrap_or_else(|| "none".to_string()); tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status"); - let module = get_module(&flow_job, &module_step); - // tracing::error!( - // "UPDATE FLOW STATUS 3: {module:#?} {unrecoverable} {} {is_last_step} {success} {skip_error_handler} is_failure_step {is_failure_step}", flow_job.canceled - // ); + let flow_value = flow_job.parse_raw_flow(); let should_continue_flow = match success { _ if stop_early => false, @@ -953,7 +937,14 @@ pub async fn update_flow_status_after_job_completion_internal< } false if next_retry( - &module.and_then(|m| m.retry.clone()).unwrap_or_default(), + flow_value + .as_ref() + .and_then(|value| match module_step { + Step::PreprocessorStep => value.preprocessor_module.as_ref().and_then(|m| m.retry.as_ref()), + Step::Step(i) => value.modules.get(i).as_ref().and_then(|m| m.retry.as_ref()), + Step::FailureStep => value.failure_module.as_ref().and_then(|m| m.retry.as_ref()), + }) + .unwrap_or(&Retry::default()), &old_status.retry, ) .is_some() => @@ -975,6 +966,7 @@ pub async fn update_flow_status_after_job_completion_internal< ( should_continue_flow, flow_job, + flow_value, stop_early, skip_if_stop_early, nresult, @@ -1042,13 +1034,11 @@ pub async fn update_flow_status_after_job_completion_internal< let args_hash = hash_args(db, client, w_id, job_id_for_status, &flow_job.args).await; let flow_path = flow_job.script_path(); - let version_hash = if let Some(rc) = flow_job.raw_flow.as_ref() { - use std::hash::Hasher; - let mut s = DefaultHasher::new(); - serde_json::to_string(&rc.0) - .unwrap_or_default() - .hash(&mut s); - format!("flow_{}", hex::encode(s.finish().to_be_bytes())) + let version_hash = if let Some(sqlx::types::Json(s)) = flow_job.raw_flow.as_ref() { + use std::hash::{Hash, Hasher}; + let mut h = DefaultHasher::new(); + s.get().hash(&mut h); + format!("flow_{}", hex::encode(h.finish().to_be_bytes())) } else { "flow_unknown".to_string() }; @@ -1107,6 +1097,7 @@ pub async fn update_flow_status_after_job_completion_internal< tracing::debug!(id = %flow_job.id, "start handle flow"); match handle_flow( flow_job.clone(), + flow_value, db, client, Some(nresult.clone()), @@ -1239,19 +1230,6 @@ async fn retrieve_flow_jobs_results( Ok(to_raw_value(&results)) } -fn get_module(flow_job: &QueuedJob, module_step: &Step) -> Option { - let raw_flow = flow_job.parse_raw_flow(); - if let Some(raw_flow) = raw_flow { - match module_step { - Step::PreprocessorStep => raw_flow.preprocessor_module.map(|x| *x.clone()), - Step::Step(i) => raw_flow.modules.get(*i).map(|x| x.clone()), - Step::FailureStep => raw_flow.failure_module.map(|x| *x.clone()), - } - } else { - None - } -} - async fn compute_skip_branchall_failure<'c>( job: &Uuid, branch: usize, @@ -1506,6 +1484,7 @@ async fn transform_input( #[instrument(level = "trace", skip_all)] pub async fn handle_flow( flow_job: Arc, + flow_value: Option, db: &sqlx::Pool, client: &AuthedClient, last_result: Option>>, @@ -1514,8 +1493,7 @@ pub async fn handle_flow( rsmq: Option, job_completed_tx: Sender, ) -> anyhow::Result<()> { - let flow = flow_job - .parse_raw_flow() + let flow = flow_value .with_context(|| "Unable to parse flow definition")?; let status = flow_job .parse_flow_status() From 44ed4045f7b7daa70b943df9405a72b07e86df12 Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Thu, 14 Nov 2024 12:25:30 +0100 Subject: [PATCH 03/31] api: cleanup job data structure (#4705) --- backend/windmill-api/src/jobs.rs | 377 ++++++++++++++-------------- backend/windmill-common/src/jobs.rs | 16 -- 2 files changed, 188 insertions(+), 205 deletions(-) diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 433c859619..680bfd6c76 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -12,6 +12,7 @@ use quick_cache::sync::Cache; use serde_json::value::RawValue; use sqlx::Pool; use std::collections::HashMap; +use std::ops::{Deref, DerefMut}; #[cfg(feature = "prometheus")] use std::sync::atomic::Ordering; use tokio::io::AsyncReadExt; @@ -600,7 +601,7 @@ async fn get_flow_job_debug_info( Extension(db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::Result { - let job = get_queued_job(&id, w_id.as_str(), &db).await?; + let job = get_queued_job_ex(&db, &w_id, id, false, None).await?; if let Some(job) = job { let is_flow = job.is_flow(); if job.is_flow_step || !is_flow { @@ -734,6 +735,67 @@ fn generate_get_job_query(no_logs: bool, table: &str) -> String { {join} WHERE id = $1 AND {table}.workspace_id = $2"); } +pub async fn get_queued_job_ex( + db: &DB, + workspace_id: &str, + job_id: Uuid, + no_logs: bool, + // first optional is if authed need to be checked, second is the opt_authed itself + opt_authed: Option<&Option>, +) -> error::Result>> { + let query = if no_logs { &*GET_QUEUED_JOB_QUERY_NO_LOGS } else { &*GET_QUEUED_JOB_QUERY }; + let job = sqlx::query_as::<_, JobExtended>(query) + .bind(job_id) + .bind(workspace_id) + .fetch_optional(db) + .await?; + + if let Some(job) = job.as_ref() { + if opt_authed.is_some_and(|x| x.is_none()) && job.created_by != "anonymous" { + return Err(Error::BadRequest( + "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), + )); + } + } + + Ok(job) +} +pub async fn get_completed_job_ex( + db: &DB, + workspace_id: &str, + job_id: Uuid, + no_logs: bool, + // first optional is if authed need to be checked, second is the opt_authed itself + opt_authed: Option<&Option>, +) -> error::Result>> { + let query = if no_logs { &*GET_COMPLETED_JOB_QUERY_NO_LOGS } else { &*GET_COMPLETED_JOB_QUERY }; + let cjob = sqlx::query_as::<_, JobExtended>(query) + .bind(job_id) + .bind(workspace_id) + .fetch_optional(db) + .await?; + + if let Some(job) = cjob.as_ref() { + if opt_authed.is_some_and(|x| x.is_none()) && job.created_by != "anonymous" { + return Err(Error::BadRequest( + "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), + )); + } + } + + if let Some(mut cjob) = cjob { + let CompletedJobWithFormattedResult { mut cj, result } = format_completed_job_result(cjob.inner); + cj.result = match result { + Some(FormattedResult::RawValue(rv)) => rv, + Some(FormattedResult::Vec(v)) => Some(to_raw_value(&v)), + None => None, + }.map(sqlx::types::Json); + cjob.inner = cj; + return Ok(Some(cjob)); + } + + Ok(cjob) +} pub async fn get_job_internal( db: &DB, workspace_id: &str, @@ -742,48 +804,18 @@ pub async fn get_job_internal( // first optional is if authed need to be checked, second is the opt_authed itself opt_authed: Option<&Option>, ) -> error::Result { - let cjob_maybe = sqlx::query_as::<_, CompletedJob>(if no_logs { - &*GET_COMPLETED_JOB_QUERY_NO_LOGS - } else { - &*GET_COMPLETED_JOB_QUERY - }) - .bind(job_id) - .bind(workspace_id) - .fetch_optional(db) - .await? - .map(Job::CompletedJob); - - if let Some(cjob) = cjob_maybe { - Ok(match cjob { - Job::CompletedJob(cjob) => { - if opt_authed.is_some_and(|x| x.is_none()) && cjob.created_by != "anonymous" { - return Err(Error::BadRequest( - "As a non logged in user, you can only see jobs ran by anonymous users" - .to_string(), - )); - } - Job::CompletedJobWithFormattedResult(format_completed_job_result(cjob)) - } - cjob => cjob, - }) - } else { - let job_o = sqlx::query_as::<_, QueuedJob>(if no_logs { - &*GET_QUEUED_JOB_QUERY_NO_LOGS - } else { - &*GET_QUEUED_JOB_QUERY - }) - .bind(job_id) - .bind(workspace_id) - .fetch_optional(db) + let cjob = get_completed_job_ex(db, workspace_id, job_id, no_logs, opt_authed.clone()) .await? - .map(Job::QueuedJob); - let job: Job = not_found_if_none(job_o, "Job", job_id.to_string())?; - if opt_authed.is_some_and(|x| x.is_none()) && job.created_by() != "anonymous" { - return Err(Error::BadRequest( - "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), - )); + .map(Job::CompletedJob); + + match cjob { + Some(cjob) => Ok(cjob), + None => { + let job_maybe = get_queued_job_ex(db, workspace_id, job_id, no_logs, opt_authed) + .await? + .map(Job::QueuedJob); + not_found_if_none(job_maybe, "Job", job_id.to_string()) } - Ok(job) } } @@ -1735,7 +1767,6 @@ async fn resume_suspended_job_internal( let trigger_email = match &parent_flow { Job::CompletedJob(job) => &job.email, Job::QueuedJob(job) => &job.email, - Job::CompletedJobWithFormattedResult(job) => &job.cj.email, }; conditionally_require_authed_user(authed.clone(), flow_status, trigger_email)?; @@ -2010,7 +2041,6 @@ pub async fn get_suspended_job_flow( let trigger_email = match &flow { Job::CompletedJob(job) => &job.email, Job::QueuedJob(job) => &job.email, - Job::CompletedJobWithFormattedResult(job) => &job.cj.email, }; conditionally_require_authed_user(authed.clone(), flow_status.clone(), trigger_email)?; @@ -2225,13 +2255,45 @@ pub async fn get_resume_urls( Ok(Json(res)) } +#[derive(sqlx::FromRow, Debug, Serialize)] +pub struct JobExtended { + #[sqlx(flatten)] + #[serde(flatten)] + inner: T, + + #[sqlx(skip)] + #[serde(skip_serializing_if = "Option::is_none")] + pub self_wait_time_ms: Option, + #[sqlx(skip)] + #[serde(skip_serializing_if = "Option::is_none")] + pub aggregate_wait_time_ms: Option, +} + +impl JobExtended { + pub fn new(self_wait_time_ms: Option, aggregate_wait_time_ms: Option, inner: T) -> Self { + Self { inner, self_wait_time_ms, aggregate_wait_time_ms } + } +} + +impl Deref for JobExtended { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for JobExtended { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + #[derive(Serialize, Debug)] #[serde(tag = "type")] pub enum Job { - QueuedJob(QueuedJob), - CompletedJob(CompletedJob), - #[serde(rename = "CompletedJob")] - CompletedJobWithFormattedResult(CompletedJobWithFormattedResult), + QueuedJob(JobExtended), + CompletedJob(JobExtended), } impl Job { @@ -2239,27 +2301,6 @@ impl Job { match self { Job::QueuedJob(job) => &job.created_by, Job::CompletedJob(job) => &job.created_by, - Job::CompletedJobWithFormattedResult(job) => &job.cj.created_by, - } - } - pub fn raw_flow(&self) -> Option { - match self { - Job::QueuedJob(job) => job - .raw_flow - .as_ref() - .map(|rf| serde_json::from_str(rf.0.get()).ok()) - .flatten(), - Job::CompletedJob(job) => job - .raw_flow - .as_ref() - .map(|rf| serde_json::from_str(rf.0.get()).ok()) - .flatten(), - Job::CompletedJobWithFormattedResult(job) => job - .cj - .raw_flow - .as_ref() - .map(|rf| serde_json::from_str(rf.0.get()).ok()) - .flatten(), } } @@ -2279,13 +2320,6 @@ impl Job { job.logs = Some(logs.to_string()); } } - Job::CompletedJobWithFormattedResult(job) => { - if let Some(ref mut l) = job.cj.logs { - l.push_str(logs); - } else { - job.cj.logs = Some(logs.to_string()); - } - } } } @@ -2293,7 +2327,6 @@ impl Job { match self { Job::QueuedJob(job) => job.logs.as_ref().map(|l| l.len()), Job::CompletedJob(job) => job.logs.as_ref().map(|l| l.len()), - Job::CompletedJobWithFormattedResult(job) => job.cj.logs.as_ref().map(|l| l.len()), } } @@ -2301,7 +2334,6 @@ impl Job { match self { Job::QueuedJob(job) => job.logs.clone(), Job::CompletedJob(job) => job.logs.clone(), - Job::CompletedJobWithFormattedResult(job) => job.cj.logs.clone(), } } pub fn flow_status(&self) -> Option { @@ -2316,19 +2348,12 @@ impl Job { .as_ref() .map(|rf| serde_json::from_str(rf.0.get()).ok()) .flatten(), - Job::CompletedJobWithFormattedResult(job) => job - .cj - .flow_status - .as_ref() - .map(|rf| serde_json::from_str(rf.0.get()).ok()) - .flatten(), } } pub fn is_flow_step(&self) -> bool { match self { Job::QueuedJob(job) => job.is_flow_step, Job::CompletedJob(job) => job.is_flow_step, - Job::CompletedJobWithFormattedResult(job) => job.cj.is_flow_step, } } @@ -2343,7 +2368,6 @@ impl Job { match self { Job::QueuedJob(job) => &job.job_kind, Job::CompletedJob(job) => &job.job_kind, - Job::CompletedJobWithFormattedResult(job) => &job.cj.job_kind, } } @@ -2351,7 +2375,6 @@ impl Job { match self { Job::QueuedJob(job) => job.id, Job::CompletedJob(job) => job.id, - Job::CompletedJobWithFormattedResult(job) => job.cj.id, } } @@ -2359,7 +2382,6 @@ impl Job { match self { Job::QueuedJob(job) => &job.workspace_id, Job::CompletedJob(job) => &job.workspace_id, - Job::CompletedJobWithFormattedResult(job) => &job.cj.workspace_id, } } @@ -2367,7 +2389,6 @@ impl Job { match self { Job::QueuedJob(job) => job.script_path.as_ref(), Job::CompletedJob(job) => job.script_path.as_ref(), - Job::CompletedJobWithFormattedResult(job) => job.cj.script_path.as_ref(), } .map(String::as_str) .unwrap_or("tmp/main") @@ -2377,7 +2398,6 @@ impl Job { match self { Job::QueuedJob(job) => job.args.as_ref(), Job::CompletedJob(job) => job.args.as_ref(), - Job::CompletedJobWithFormattedResult(job) => job.cj.args.as_ref(), } } @@ -2426,10 +2446,6 @@ impl Job { job.self_wait_time_ms = self_wait_time; job.aggregate_wait_time_ms = aggregate_wait_time; } - Job::CompletedJobWithFormattedResult(job) => { - job.cj.self_wait_time_ms = self_wait_time; - job.cj.aggregate_wait_time_ms = aggregate_wait_time; - } } Ok(()) } @@ -2557,86 +2573,82 @@ impl UnifiedJob { impl<'a> From for Job { fn from(uj: UnifiedJob) -> Self { match uj.typ.as_ref() { - "CompletedJob" => Job::CompletedJob(CompletedJob { - workspace_id: uj.workspace_id, - id: uj.id, - parent_job: uj.parent_job, - created_by: uj.created_by, - created_at: uj.created_at, - started_at: uj.started_at.unwrap_or(uj.created_at), - duration_ms: uj.duration_ms.unwrap(), - success: uj.success.unwrap(), - script_hash: uj.script_hash, - script_path: uj.script_path, - args: None, - result: None, - logs: None, - flow_status: None, - deleted: uj.deleted, - canceled: uj.canceled, - canceled_by: uj.canceled_by, - raw_code: None, - canceled_reason: None, - job_kind: uj.job_kind, - schedule_path: uj.schedule_path, - permissioned_as: uj.permissioned_as, - raw_flow: None, - is_flow_step: uj.is_flow_step, - language: uj.language, - is_skipped: uj.is_skipped, - email: uj.email, - visible_to_owner: uj.visible_to_owner, - mem_peak: uj.mem_peak, - tag: uj.tag, - priority: uj.priority, - labels: uj.labels, - self_wait_time_ms: uj.self_wait_time_ms, - aggregate_wait_time_ms: uj.aggregate_wait_time_ms, - }), - "QueuedJob" => Job::QueuedJob(QueuedJob { - workspace_id: uj.workspace_id, - id: uj.id, - parent_job: uj.parent_job, - created_by: uj.created_by, - created_at: uj.created_at, - started_at: uj.started_at, - script_hash: uj.script_hash, - script_path: uj.script_path, - args: None, - running: uj.running.unwrap(), - scheduled_for: uj.scheduled_for.unwrap(), - logs: None, - flow_status: None, - raw_code: None, - raw_lock: None, - canceled: uj.canceled, - canceled_by: uj.canceled_by, - canceled_reason: None, - last_ping: None, - job_kind: uj.job_kind, - schedule_path: uj.schedule_path, - permissioned_as: uj.permissioned_as, - raw_flow: None, - is_flow_step: uj.is_flow_step, - language: uj.language, - same_worker: false, - pre_run_error: None, - email: uj.email, - visible_to_owner: uj.visible_to_owner, - suspend: uj.suspend, - mem_peak: uj.mem_peak, - root_job: None, - leaf_jobs: None, - tag: uj.tag, - concurrent_limit: uj.concurrent_limit, - concurrency_time_window_s: uj.concurrency_time_window_s, - timeout: None, - flow_step_id: None, - cache_ttl: None, - priority: uj.priority, - self_wait_time_ms: uj.self_wait_time_ms, - aggregate_wait_time_ms: uj.aggregate_wait_time_ms, - }), + "CompletedJob" => Job::CompletedJob(JobExtended::new(uj.self_wait_time_ms, uj.aggregate_wait_time_ms, CompletedJob { + workspace_id: uj.workspace_id, + id: uj.id, + parent_job: uj.parent_job, + created_by: uj.created_by, + created_at: uj.created_at, + started_at: uj.started_at.unwrap_or(uj.created_at), + duration_ms: uj.duration_ms.unwrap(), + success: uj.success.unwrap(), + script_hash: uj.script_hash, + script_path: uj.script_path, + args: None, + result: None, + logs: None, + flow_status: None, + deleted: uj.deleted, + canceled: uj.canceled, + canceled_by: uj.canceled_by, + raw_code: None, + canceled_reason: None, + job_kind: uj.job_kind, + schedule_path: uj.schedule_path, + permissioned_as: uj.permissioned_as, + raw_flow: None, + is_flow_step: uj.is_flow_step, + language: uj.language, + is_skipped: uj.is_skipped, + email: uj.email, + visible_to_owner: uj.visible_to_owner, + mem_peak: uj.mem_peak, + tag: uj.tag, + priority: uj.priority, + labels: uj.labels, + })), + "QueuedJob" => Job::QueuedJob(JobExtended::new(uj.self_wait_time_ms, uj.aggregate_wait_time_ms, QueuedJob { + workspace_id: uj.workspace_id, + id: uj.id, + parent_job: uj.parent_job, + created_by: uj.created_by, + created_at: uj.created_at, + started_at: uj.started_at, + script_hash: uj.script_hash, + script_path: uj.script_path, + args: None, + running: uj.running.unwrap(), + scheduled_for: uj.scheduled_for.unwrap(), + logs: None, + flow_status: None, + raw_code: None, + raw_lock: None, + canceled: uj.canceled, + canceled_by: uj.canceled_by, + canceled_reason: None, + last_ping: None, + job_kind: uj.job_kind, + schedule_path: uj.schedule_path, + permissioned_as: uj.permissioned_as, + raw_flow: None, + is_flow_step: uj.is_flow_step, + language: uj.language, + same_worker: false, + pre_run_error: None, + email: uj.email, + visible_to_owner: uj.visible_to_owner, + suspend: uj.suspend, + mem_peak: uj.mem_peak, + root_job: None, + leaf_jobs: None, + tag: uj.tag, + concurrent_limit: uj.concurrent_limit, + concurrency_time_window_s: uj.concurrency_time_window_s, + timeout: None, + flow_step_id: None, + cache_ttl: None, + priority: uj.priority, + })), t => panic!("job type {} not valid", t), } } @@ -3079,7 +3091,7 @@ pub async fn run_workflow_as_code( i += 1; } - let job = get_queued_job(&job_id, &w_id, &db).await?; + let job = get_queued_job_ex(&db, &w_id, job_id, true, None).await?; if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); @@ -3087,6 +3099,7 @@ pub async fn run_workflow_as_code( } let job = not_found_if_none(job, "Queued Job", &job_id.to_string())?; + let JobExtended { inner: job, .. } = job; let (job_payload, tag, _delete_after_use, timeout) = match job.job_kind { JobKind::Preview => ( JobPayload::Code(RawCode { @@ -5058,24 +5071,10 @@ async fn get_completed_job<'a>( Extension(db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::Result { - let job_o = sqlx::query_as::<_, CompletedJob>("SELECT id, workspace_id, parent_job, created_by, created_at, duration_ms, success, script_hash, script_path, - CASE WHEN args is null or pg_column_size(args) < 90000 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args, CASE WHEN result is null or pg_column_size(result) < 90000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind, - schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language, started_at, is_skipped, - raw_lock, email, visible_to_owner, mem_peak, tag, priority, result->'wm_labels' as labels FROM completed_job WHERE id = $1 AND workspace_id = $2") - .bind(id) - .bind(&w_id) - .fetch_optional(&db) + let job_o = get_completed_job_ex(&db, &w_id, id, false, Some(&opt_authed)) .await?; let cj = not_found_if_none(job_o, "Completed Job", id.to_string())?; - - if opt_authed.is_none() && cj.created_by != "anonymous" { - return Err(Error::BadRequest( - "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), - )); - } - let cj = format_completed_job_result(cj); - let response = Json(cj).into_response(); // let extra_log = query_scalar!( // "SELECT substr(logs, $1) as logs FROM large_logs WHERE workspace_id = $2 AND job_id = $3", diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 6feceb8d51..37bc220b94 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -108,13 +108,6 @@ pub struct QueuedJob { pub cache_ttl: Option, #[serde(skip_serializing_if = "Option::is_none")] pub priority: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - #[sqlx(skip)] - pub self_wait_time_ms: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[sqlx(skip)] - pub aggregate_wait_time_ms: Option, } impl QueuedJob { @@ -198,8 +191,6 @@ impl Default for QueuedJob { flow_step_id: None, cache_ttl: None, priority: None, - self_wait_time_ms: None, - aggregate_wait_time_ms: None, } } } @@ -253,13 +244,6 @@ pub struct CompletedJob { pub priority: Option, #[serde(skip_serializing_if = "Option::is_none")] pub labels: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - #[sqlx(skip)] - pub self_wait_time_ms: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[sqlx(skip)] - pub aggregate_wait_time_ms: Option, } impl CompletedJob { From 50ff183baeaa2a39c2b0188ee8dd6172b08ae801 Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Thu, 14 Nov 2024 12:33:50 +0100 Subject: [PATCH 04/31] feat(backend): monitor minimal version of living workers (#4704) --- ...1775c67f075c8bc83458e8be2e242347218d6.json | 22 +++++++++++++++ backend/Cargo.lock | 1 + backend/Cargo.toml | 1 + backend/src/main.rs | 4 +-- backend/src/monitor.rs | 9 +++++-- backend/windmill-api/src/lib.rs | 6 +---- backend/windmill-common/Cargo.toml | 1 + backend/windmill-common/src/utils.rs | 5 ++++ backend/windmill-common/src/worker.rs | 27 +++++++++++++++++++ 9 files changed, 66 insertions(+), 10 deletions(-) create mode 100644 backend/.sqlx/query-ad03e5acf10ef94abc37cb9f56b1775c67f075c8bc83458e8be2e242347218d6.json diff --git a/backend/.sqlx/query-ad03e5acf10ef94abc37cb9f56b1775c67f075c8bc83458e8be2e242347218d6.json b/backend/.sqlx/query-ad03e5acf10ef94abc37cb9f56b1775c67f075c8bc83458e8be2e242347218d6.json new file mode 100644 index 0000000000..9304f1c75d --- /dev/null +++ b/backend/.sqlx/query-ad03e5acf10ef94abc37cb9f56b1775c67f075c8bc83458e8be2e242347218d6.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "wm_version", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ad03e5acf10ef94abc37cb9f56b1775c67f075c8bc83458e8be2e242347218d6" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d37f3cb23a..f239021c99 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10811,6 +10811,7 @@ dependencies = [ "rand 0.8.5", "regex", "reqwest 0.12.9", + "semver 1.0.23", "serde", "serde_json", "sha2 0.10.8", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 800e165e33..51bf578c96 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -180,6 +180,7 @@ tokio-util = { version = "^0", features = ["io"] } json-pointer = "^0" itertools = "^0" regex = "^1" +semver = "^1" deno_fetch = "0.195.0" deno_tls = "0.158.0" diff --git a/backend/src/main.rs b/backend/src/main.rs index 6869a24647..1f1fab319d 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -7,7 +7,6 @@ */ use anyhow::Context; -use git_version::git_version; use monitor::{ reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store, @@ -42,7 +41,7 @@ use windmill_common::{ }, scripts::ScriptLang, stats_ee::schedule_stats, - utils::{hostname, rd_string, Mode}, + utils::{hostname, rd_string, Mode, GIT_VERSION}, worker::{reload_custom_tags_setting, HUB_CACHE_DIR, TMP_DIR, WORKER_GROUP}, DB, METRICS_ENABLED, }; @@ -84,7 +83,6 @@ use crate::monitor::{ #[cfg(feature = "parquet")] use crate::monitor::reload_s3_cache_setting; -const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); const DEFAULT_NUM_WORKERS: usize = 1; const DEFAULT_PORT: u16 = 8000; const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index da811bb6ff..b40c89c766 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -51,8 +51,8 @@ use windmill_common::{ utils::{now_from_db, rd_string, report_critical_error, Mode}, worker::{ load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting, - DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG, WORKER_CONFIG, - WORKER_GROUP, + update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG, + WORKER_CONFIG, WORKER_GROUP, }, BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, CRITICAL_ALERT_MUTE_UI_ENABLED @@ -1080,6 +1080,10 @@ pub async fn monitor_db( } }; + let update_min_worker_version_f = async { + update_min_version(db).await; + }; + join!( expired_items_f, zombie_jobs_f, @@ -1088,6 +1092,7 @@ pub async fn monitor_db( worker_groups_alerts_f, jobs_waiting_alerts_f, apply_autoscaling_f, + update_min_worker_version_f, ); } diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 5a7f7fcded..fe74584d1b 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -25,7 +25,6 @@ use argon2::Argon2; use axum::extract::DefaultBodyLimit; use axum::{middleware::from_extractor, routing::get, Extension, Router}; use db::DB; -use git_version::git_version; use http::HeaderValue; use reqwest::Client; use std::collections::HashMap; @@ -40,7 +39,7 @@ use tower_http::{ }; use windmill_common::db::UserDB; use windmill_common::worker::{ALL_TAGS, CLOUD_HOSTED}; -use windmill_common::{BASE_URL, INSTANCE_NAME}; +use windmill_common::{BASE_URL, INSTANCE_NAME, utils::GIT_VERSION}; use crate::scim_ee::has_scim_token; use windmill_common::error::AppError; @@ -93,9 +92,6 @@ mod workers; mod workspaces; mod workspaces_ee; -pub const GIT_VERSION: &str = - git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); - pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB lazy_static::lazy_static! { diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index f3adbfde84..fd0f3aa8c1 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -58,6 +58,7 @@ async-stream.workspace = true const_format.workspace = true crc.workspace = true windmill-macros.workspace = true +semver.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = { optional = true, workspace = true } diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 18c0c93205..3223ffb75f 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -21,6 +21,7 @@ use reqwest::Client; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use sqlx::{Pool, Postgres}; +use semver::Version; pub const MAX_PER_PAGE: usize = 10000; pub const DEFAULT_PER_PAGE: usize = 1000; @@ -37,6 +38,10 @@ lazy_static::lazy_static! { .timeout(std::time::Duration::from_secs(20)) .connect_timeout(std::time::Duration::from_secs(10)) .build().unwrap(); + pub static ref GIT_SEM_VERSION: Version = Version::parse( + // skip first `v` character. + GIT_VERSION.split_at(1).1 + ).unwrap_or(Version::new(0, 1, 0)); } #[derive(Deserialize, Clone)] diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 37ae61c8d6..eb017bc79f 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1,6 +1,7 @@ use const_format::concatcp; use itertools::Itertools; use regex::Regex; +use semver::Version; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::{ @@ -88,6 +89,7 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); + pub static ref MIN_VERSION: Arc> = Arc::new(RwLock::new(Version::new(0, 0, 0))); } pub async fn make_suspended_pull_query(wc: &WorkerConfig) { @@ -548,6 +550,31 @@ pub fn get_windmill_memory_usage() -> Option { } } +pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(executor: E) -> bool { + use crate::utils::{GIT_VERSION, GIT_SEM_VERSION}; + + // fetch all pings with a different version than self from the last 5 minutes. + let pings = sqlx::query_scalar!( + "SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'", + GIT_VERSION + ).fetch_all(executor).await.unwrap_or_default(); + + let cur_version = GIT_SEM_VERSION.clone(); + let min_version = pings + .iter() + .filter(|x| !x.is_empty()) + .filter_map(|x| semver::Version::parse(x.split_at(1).1).ok()) + .min() + .unwrap_or_else(|| cur_version.clone()); + + if min_version != cur_version { + tracing::info!("Minimal worker version: {min_version}"); + } + + *MIN_VERSION.write().await = min_version.clone(); + min_version >= cur_version +} + pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) { let (tags, dw) = { let wc = WORKER_CONFIG.read().await.clone(); From 2a1bff3160b65eadba399229b5f53950ec2c0d48 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 14 Nov 2024 13:12:40 +0100 Subject: [PATCH 05/31] feat: allow setting password and login type from superadmin UI --- ...76f06c6077c619c0ab1ac4edf633b22df98c3.json | 23 ----- ...67e6eb876792ec8f83e9b03c2fb46bb12e0b9.json | 15 +++ ...087775a721a6ee7ae35b03fd4ce3563ea3838.json | 23 ----- ...610c021a0a5b63ffe5afd7878385c8af6bc6c.json | 24 ----- backend/ee-repo-ref.txt | 2 +- backend/windmill-api/openapi.yaml | 65 +++++++++++++ backend/windmill-api/src/users.rs | 56 ++++++++++- backend/windmill-api/src/users_ee.rs | 1 + .../ChangeInstanceUsernameInner.svelte | 6 +- .../lib/components/InstanceNameEditor.svelte | 93 ++++++++++++++++++- .../lib/components/SuperadminSettings.svelte | 4 + 11 files changed, 234 insertions(+), 78 deletions(-) delete mode 100644 backend/.sqlx/query-0f04f42a596b6759a84ba7d26ea76f06c6077c619c0ab1ac4edf633b22df98c3.json create mode 100644 backend/.sqlx/query-349396e8fdd96d45875110bc06767e6eb876792ec8f83e9b03c2fb46bb12e0b9.json delete mode 100644 backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json delete mode 100644 backend/.sqlx/query-cd40b9c552d76664a552457c4a9610c021a0a5b63ffe5afd7878385c8af6bc6c.json diff --git a/backend/.sqlx/query-0f04f42a596b6759a84ba7d26ea76f06c6077c619c0ab1ac4edf633b22df98c3.json b/backend/.sqlx/query-0f04f42a596b6759a84ba7d26ea76f06c6077c619c0ab1ac4edf633b22df98c3.json deleted file mode 100644 index bf47f131c4..0000000000 --- a/backend/.sqlx/query-0f04f42a596b6759a84ba7d26ea76f06c6077c619c0ab1ac4edf633b22df98c3.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT flow_status AS \"_id!: Json>\" FROM queue WHERE id = $1 AND workspace_id = $2 LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "_id!: Json>", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "0f04f42a596b6759a84ba7d26ea76f06c6077c619c0ab1ac4edf633b22df98c3" -} diff --git a/backend/.sqlx/query-349396e8fdd96d45875110bc06767e6eb876792ec8f83e9b03c2fb46bb12e0b9.json b/backend/.sqlx/query-349396e8fdd96d45875110bc06767e6eb876792ec8f83e9b03c2fb46bb12e0b9.json new file mode 100644 index 0000000000..032f287093 --- /dev/null +++ b/backend/.sqlx/query-349396e8fdd96d45875110bc06767e6eb876792ec8f83e9b03c2fb46bb12e0b9.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE password SET login_type = $1 WHERE email = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "349396e8fdd96d45875110bc06767e6eb876792ec8f83e9b03c2fb46bb12e0b9" +} diff --git a/backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json b/backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json deleted file mode 100644 index f193758557..0000000000 --- a/backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838" -} diff --git a/backend/.sqlx/query-cd40b9c552d76664a552457c4a9610c021a0a5b63ffe5afd7878385c8af6bc6c.json b/backend/.sqlx/query-cd40b9c552d76664a552457c4a9610c021a0a5b63ffe5afd7878385c8af6bc6c.json deleted file mode 100644 index 382caea08a..0000000000 --- a/backend/.sqlx/query-cd40b9c552d76664a552457c4a9610c021a0a5b63ffe5afd7878385c8af6bc6c.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT raw_flow->'modules'->($1)::int AS \"_id: Json>\" FROM queue WHERE id = $2 AND workspace_id = $3 LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "_id: Json>", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "cd40b9c552d76664a552457c4a9610c021a0a5b63ffe5afd7878385c8af6bc6c" -} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index f3e6e24f4e..498ca96e9b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -e3df64af864f9d3546a97ff21f719c553e34ff1c \ No newline at end of file +0d9c8813acd28848515c736e7b684220b5a785a3 \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 97d98d3d6b..2f59c0c91c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -283,6 +283,71 @@ paths: text/plain: schema: type: string + + /users/set_password_of/{user}: + post: + summary: set password for a specific user (require super admin) + operationId: setPasswordForUser + tags: + - user + parameters: + - name: user + in: path + required: true + schema: + type: string + requestBody: + description: set password + required: true + content: + application/json: + schema: + type: object + properties: + password: + type: string + required: + - password + responses: + "200": + description: password set + content: + text/plain: + schema: + type: string + + /users/set_login_type/{user}: + post: + summary: set login type for a specific user (require super admin) + operationId: setLoginTypeForUser + tags: + - user + parameters: + - name: user + in: path + required: true + schema: + type: string + requestBody: + description: set login type + required: true + content: + application/json: + schema: + type: object + properties: + login_type: + type: string + required: + - login_type + responses: + "200": + description: login type set + content: + text/plain: + schema: + type: string + /users/create: post: diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 380084d524..bf95a632b1 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -90,6 +90,9 @@ pub fn global_service() -> Router { .route("/accept_invite", post(accept_invite)) .route("/list_as_super_admin", get(list_users_as_super_admin)) .route("/setpassword", post(set_password)) + .route("/set_password_of/:user", post(set_password_of_user)) + .route("/set_login_type/:user", post(set_login_type)) + .route("/create", post(create_user)) .route("/update/:user", post(update_user)) .route("/delete/:user", delete(delete_user)) @@ -866,6 +869,12 @@ pub struct EditPassword { pub password: String, } +#[derive(Deserialize)] +pub struct EditLoginType { + pub login_type: String, +} + + #[derive(FromRow, Serialize)] pub struct TruncatedToken { pub label: Option, @@ -2028,7 +2037,52 @@ async fn set_password( authed: ApiAuthed, Json(ep): Json, ) -> Result { - crate::users_ee::set_password(db, argon2, authed, ep).await + let email = authed.email.clone(); + crate::users_ee::set_password(db, argon2, authed, &email, ep).await +} + +async fn set_password_of_user( + Extension(db): Extension, + Extension(argon2): Extension>>, + Path(email): Path, + authed: ApiAuthed, + Json(ep): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + crate::users_ee::set_password(db, argon2, authed, &email, ep).await +} + +async fn set_login_type( + Extension(db): Extension, + Path(email): Path, + authed: ApiAuthed, + Json(et): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + sqlx::query!( + "UPDATE password SET login_type = $1 WHERE email = $2", + et.login_type, + email + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.set_login_type", + ActionKind::Update, + "global", + Some(&email), + None, + ) + .await?; + + tx.commit().await?; + Ok(format!("login type of {} updated to {}", email, et.login_type)) + } async fn login( diff --git a/backend/windmill-api/src/users_ee.rs b/backend/windmill-api/src/users_ee.rs index e0f15a9317..65a2a2ed17 100644 --- a/backend/windmill-api/src/users_ee.rs +++ b/backend/windmill-api/src/users_ee.rs @@ -27,6 +27,7 @@ pub async fn set_password( _db: DB, _argon2: Arc>, _authed: ApiAuthed, + _user_email: &str, _ep: EditPassword, ) -> Result { Err(Error::InternalErr( diff --git a/frontend/src/lib/components/ChangeInstanceUsernameInner.svelte b/frontend/src/lib/components/ChangeInstanceUsernameInner.svelte index a321f4ac1d..1690dcd4f3 100644 --- a/frontend/src/lib/components/ChangeInstanceUsernameInner.svelte +++ b/frontend/src/lib/components/ChangeInstanceUsernameInner.svelte @@ -72,11 +72,13 @@
+ {#if isConflict} {isConflict ? 'Fix username conflict' : 'Change username'}Fix username conflict + {/if} - {isConflict ? 'Auto-generated instance username' : 'New username'} {/if} diff --git a/frontend/src/lib/components/SuperadminSettings.svelte b/frontend/src/lib/components/SuperadminSettings.svelte index 1670a51086..e47466a02a 100644 --- a/frontend/src/lib/components/SuperadminSettings.svelte +++ b/frontend/src/lib/components/SuperadminSettings.svelte @@ -276,9 +276,13 @@
{ + listUsers(activeOnly) + }} on:save={(e) => { updateName(e.detail, email) }} From 556b4a41a1d010bb8a1796f485d343c1e412d278 Mon Sep 17 00:00:00 2001 From: dieriba Date: Thu, 14 Nov 2024 13:54:21 +0100 Subject: [PATCH 06/31] feat: Support mistral anthropic for ai (#4692) * wip: openai proxy and other ai proxy integration * fixing migration script * wip: support different ai provider in front, fix proxy openai * wip: adding frontend ai provider * updated copilot types * wip: working on anthropic integration * done AI proxy front * adding new type and support for anthropic * updating gitignore * adding streaming response * added streaming prompt * push lib/gen * wip: fixing anthropic * anthropic fully supported * fix backend missing var error and fully support stream event for anthropic * remove gen directory * fixing openapi file * add support for mistral, and update create workspace components * remove deref.json * remove package-json * openapi * fix ui enable code * added utility function for init workspace ai provider * fix workspace switch bug * update anthropic property and fixed frontend error * fix workspace settings * update error message and fix typo migration file * chore: update openapi file * fix dev file * add .sqlx * all * update sqlx --------- Co-authored-by: dieriba Co-authored-by: Ruben Fiszel Co-authored-by: Ruben Fiszel --- .gitignore | 2 +- backend/.gitignore | 2 +- ...68c7b43b3a9a9efa92b5517132e9c0d8a25b.json} | 8 +- ...11eb68ab4599d64e5e5af639a9ac5d791fd0.json} | 8 +- ...6f17c8d9d0e657f28276228fc90d3e22e1304.json | 16 - ...a0100b933b7f199d41cc2f554d57c811d55f3.json | 16 + ...c61296a3ff7489ae12f52a19f9543173ac597.json | 4 +- ...06a4243c66787ab8759a8045e60effd2fb77.json} | 4 +- ...dcc40f463cbc52d94ed9315cf9a547d4c89f2.json | 4 +- ...02278974623f7a7bcad6891d7abadf4d3ea03.json | 15 - ...d964b199604ba66dedbe9f89cea727d726a2d.json | 15 + ...7db1f054ec751855c8db7352b0e9c20b63ef8.json | 22 + ...d252a14d3a2acfadcd9cd6ba27ca9799f4706.json | 22 - ...path_column_in_workspace_settings.down.sql | 7 + ...e_path_column_in_workspace_settings.up.sql | 7 + backend/windmill-api/openapi.yaml | 27 +- backend/windmill-api/src/ai.rs | 507 ++++++++++++++++++ backend/windmill-api/src/lib.rs | 6 +- backend/windmill-api/src/openai.rs | 344 ------------ backend/windmill-api/src/workspaces.rs | 57 +- backend/windmill-common/src/error.rs | 4 +- frontend/package-lock.json | 44 ++ frontend/package.json | 2 + frontend/src/lib/components/Dev.svelte | 7 +- frontend/src/lib/components/Editor.svelte | 7 +- .../src/lib/components/FlowBuilder.svelte | 16 +- .../auditLogs/AuditLogsFilters.svelte | 2 +- .../copilot/CodeCompletionStatus.svelte | 2 +- .../src/lib/components/copilot/CronGen.svelte | 36 +- .../copilot/FlowCopilotStatus.svelte | 4 +- .../lib/components/copilot/IteratorGen.svelte | 13 +- .../lib/components/copilot/MetadataGen.svelte | 14 +- .../components/copilot/PredicateGen.svelte | 9 +- .../lib/components/copilot/RegexGen.svelte | 10 +- .../lib/components/copilot/ScriptFix.svelte | 12 +- .../lib/components/copilot/ScriptGen.svelte | 28 +- .../src/lib/components/copilot/StepGen.svelte | 2 +- .../components/copilot/StepInputGen.svelte | 13 +- .../components/copilot/StepInputsGen.svelte | 13 +- ...{TestOpenaiKey.svelte => TestAiKey.svelte} | 6 +- .../src/lib/components/copilot/completion.ts | 7 +- frontend/src/lib/components/copilot/flow.ts | 29 +- frontend/src/lib/components/copilot/lib.ts | 463 +++++++++++++--- .../flows/content/FlowInputsQuick.svelte | 2 +- .../flows/map/FlowCopilotButton.svelte | 6 +- .../flows/map/FlowModuleSchemaMap.svelte | 5 +- .../components/sidebar/WorkspaceMenu.svelte | 3 +- frontend/src/lib/stores.ts | 6 +- .../src/routes/(root)/(logged)/+layout.svelte | 8 +- .../user/(user)/create_workspace/+page.svelte | 63 ++- .../(logged)/workspace_settings/+page.svelte | 64 ++- 51 files changed, 1326 insertions(+), 667 deletions(-) rename backend/.sqlx/{query-ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee.json => query-0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b.json} (56%) rename backend/.sqlx/{query-188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3.json => query-0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0.json} (79%) delete mode 100644 backend/.sqlx/query-034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304.json create mode 100644 backend/.sqlx/query-1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3.json rename backend/.sqlx/{query-acdaa5151f8f7f37bb8c8c5a7d146789887e47db9695fc26b1dfaedd735e1e60.json => query-286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77.json} (60%) delete mode 100644 backend/.sqlx/query-6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03.json create mode 100644 backend/.sqlx/query-6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d.json create mode 100644 backend/.sqlx/query-ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8.json delete mode 100644 backend/.sqlx/query-f8b689ec3e09f14dc02ac9e9f98d252a14d3a2acfadcd9cd6ba27ca9799f4706.json create mode 100644 backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.down.sql create mode 100644 backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.up.sql create mode 100644 backend/windmill-api/src/ai.rs delete mode 100644 backend/windmill-api/src/openai.rs rename frontend/src/lib/components/copilot/{TestOpenaiKey.svelte => TestAiKey.svelte} (87%) diff --git a/.gitignore b/.gitignore index ffda06e6cb..1e59be575c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ frontend/src/routes/test.svelte CaddyfileRemoteMalo *.swp **/.idea/ -.direnv +.direnv \ No newline at end of file diff --git a/backend/.gitignore b/backend/.gitignore index 7900ae5fc2..4ca669fc53 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -2,7 +2,7 @@ target/ .env oauth.json oauth2.json -windmill-api/openapi-deref.yaml tracing.folded heaptrack* index/ +windmill-api/openapi-*.* \ No newline at end of file diff --git a/backend/.sqlx/query-ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee.json b/backend/.sqlx/query-0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b.json similarity index 56% rename from backend/.sqlx/query-ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee.json rename to backend/.sqlx/query-0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b.json index 30b5230f96..e38ddcb726 100644 --- a/backend/.sqlx/query-ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee.json +++ b/backend/.sqlx/query-0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b.json @@ -1,12 +1,12 @@ { "db_name": "PostgreSQL", - "query": "SELECT openai_resource_path, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1", + "query": "SELECT ai_resource, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1", "describe": { "columns": [ { "ordinal": 0, - "name": "openai_resource_path", - "type_info": "Varchar" + "name": "ai_resource", + "type_info": "Jsonb" }, { "ordinal": 1, @@ -24,5 +24,5 @@ false ] }, - "hash": "ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee" + "hash": "0230bd64d2b6719c3536a106e18a68c7b43b3a9a9efa92b5517132e9c0d8a25b" } diff --git a/backend/.sqlx/query-188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3.json b/backend/.sqlx/query-0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0.json similarity index 79% rename from backend/.sqlx/query-188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3.json rename to backend/.sqlx/query-0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0.json index 83139e3a86..5d0a70183e 100644 --- a/backend/.sqlx/query-188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3.json +++ b/backend/.sqlx/query-0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n -- slack_team_id, \n -- slack_name, \n -- slack_command_script, \n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n webhook, \n deploy_to, \n error_handler, \n openai_resource_path, \n code_completion_enabled, \n error_handler_extra_args, \n error_handler_muted_on_cancel, \n large_file_storage, \n git_sync,\n default_app,\n default_scripts,\n workspace.name\n FROM workspace_settings\n LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n WHERE workspace_id = $1", + "query": "SELECT\n -- slack_team_id, \n -- slack_name, \n -- slack_command_script, \n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n webhook, \n deploy_to, \n error_handler, \n ai_resource, \n code_completion_enabled, \n error_handler_extra_args, \n error_handler_muted_on_cancel, \n large_file_storage, \n git_sync,\n default_app,\n default_scripts,\n workspace.name\n FROM workspace_settings\n LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n WHERE workspace_id = $1", "describe": { "columns": [ { @@ -35,8 +35,8 @@ }, { "ordinal": 6, - "name": "openai_resource_path", - "type_info": "Varchar" + "name": "ai_resource", + "type_info": "Jsonb" }, { "ordinal": 7, @@ -102,5 +102,5 @@ false ] }, - "hash": "188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3" + "hash": "0331a81262e2d3c1bcfaeb64617b11eb68ab4599d64e5e5af639a9ac5d791fd0" } diff --git a/backend/.sqlx/query-034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304.json b/backend/.sqlx/query-034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304.json deleted file mode 100644 index e17ea413db..0000000000 --- a/backend/.sqlx/query-034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE workspace_settings SET openai_resource_path = $1, code_completion_enabled = $2 WHERE workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Bool", - "Text" - ] - }, - "nullable": [] - }, - "hash": "034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304" -} diff --git a/backend/.sqlx/query-1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3.json b/backend/.sqlx/query-1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3.json new file mode 100644 index 0000000000..9b94af8432 --- /dev/null +++ b/backend/.sqlx/query-1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET ai_resource = $1, code_completion_enabled = $2 WHERE workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1610c79b8d238e3a35d795f34a2a0100b933b7f199d41cc2f554d57c811d55f3" +} diff --git a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json index 16a701fdff..6189a1fdab 100644 --- a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json +++ b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json @@ -65,8 +65,8 @@ }, { "ordinal": 12, - "name": "openai_resource_path", - "type_info": "Varchar" + "name": "ai_resource", + "type_info": "Jsonb" }, { "ordinal": 13, diff --git a/backend/.sqlx/query-acdaa5151f8f7f37bb8c8c5a7d146789887e47db9695fc26b1dfaedd735e1e60.json b/backend/.sqlx/query-286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77.json similarity index 60% rename from backend/.sqlx/query-acdaa5151f8f7f37bb8c8c5a7d146789887e47db9695fc26b1dfaedd735e1e60.json rename to backend/.sqlx/query-286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77.json index 90b77d6cb7..790948f378 100644 --- a/backend/.sqlx/query-acdaa5151f8f7f37bb8c8c5a7d146789887e47db9695fc26b1dfaedd735e1e60.json +++ b/backend/.sqlx/query-286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT value\n FROM resource\n WHERE path = $1 AND workspace_id = $2", + "query": "SELECT value\n FROM resource\n WHERE path = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -19,5 +19,5 @@ true ] }, - "hash": "acdaa5151f8f7f37bb8c8c5a7d146789887e47db9695fc26b1dfaedd735e1e60" + "hash": "286fa00c088df146c08c1934556f06a4243c66787ab8759a8045e60effd2fb77" } diff --git a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json b/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json index 5f4f137cd9..cfef50c160 100644 --- a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json +++ b/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json @@ -65,8 +65,8 @@ }, { "ordinal": 12, - "name": "openai_resource_path", - "type_info": "Varchar" + "name": "ai_resource", + "type_info": "Jsonb" }, { "ordinal": 13, diff --git a/backend/.sqlx/query-6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03.json b/backend/.sqlx/query-6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03.json deleted file mode 100644 index fe7c2cf5c3..0000000000 --- a/backend/.sqlx/query-6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE workspace_settings SET openai_resource_path = NULL, code_completion_enabled = $1 WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Bool", - "Text" - ] - }, - "nullable": [] - }, - "hash": "6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03" -} diff --git a/backend/.sqlx/query-6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d.json b/backend/.sqlx/query-6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d.json new file mode 100644 index 0000000000..4891854d48 --- /dev/null +++ b/backend/.sqlx/query-6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET ai_resource = NULL, code_completion_enabled = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "6940ddc5ee8d2a1048213d46f52d964b199604ba66dedbe9f89cea727d726a2d" +} diff --git a/backend/.sqlx/query-ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8.json b/backend/.sqlx/query-ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8.json new file mode 100644 index 0000000000..a309a03762 --- /dev/null +++ b/backend/.sqlx/query-ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ai_resource FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ai_resource", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8" +} diff --git a/backend/.sqlx/query-f8b689ec3e09f14dc02ac9e9f98d252a14d3a2acfadcd9cd6ba27ca9799f4706.json b/backend/.sqlx/query-f8b689ec3e09f14dc02ac9e9f98d252a14d3a2acfadcd9cd6ba27ca9799f4706.json deleted file mode 100644 index 0e42f60a93..0000000000 --- a/backend/.sqlx/query-f8b689ec3e09f14dc02ac9e9f98d252a14d3a2acfadcd9cd6ba27ca9799f4706.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT openai_resource_path FROM workspace_settings WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "openai_resource_path", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "f8b689ec3e09f14dc02ac9e9f98d252a14d3a2acfadcd9cd6ba27ca9799f4706" -} diff --git a/backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.down.sql b/backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.down.sql new file mode 100644 index 0000000000..b0ee6079b0 --- /dev/null +++ b/backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.down.sql @@ -0,0 +1,7 @@ +-- Add down migration script here +ALTER TABLE workspace_settings +ALTER COLUMN ai_resource TYPE text +USING ai_resource->>'path'; + +ALTER TABLE workspace_settings +RENAME COLUMN ai_resource to openai_resource_path; \ No newline at end of file diff --git a/backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.up.sql b/backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.up.sql new file mode 100644 index 0000000000..7288b0f13e --- /dev/null +++ b/backend/migrations/20241105133531_alter_openai_resource_path_column_in_workspace_settings.up.sql @@ -0,0 +1,7 @@ +-- Add up migration script here +ALTER TABLE workspace_settings +ALTER COLUMN openai_resource_path TYPE jsonb +USING jsonb_build_object('provider', 'openai', 'path', openai_resource_path); + +ALTER TABLE workspace_settings +RENAME COLUMN openai_resource_path TO ai_resource; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 2f59c0c91c..ade446bf67 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1681,8 +1681,8 @@ paths: type: string deploy_to: type: string - openai_resource_path: - type: string + ai_resource: + $ref: "#/components/schemas/AiResource" code_completion_enabled: type: boolean error_handler: @@ -1966,8 +1966,8 @@ paths: required: - code_completion_enabled properties: - openai_resource_path: - type: string + ai_resource: + $ref: "#/components/schemas/AiResource" code_completion_enabled: type: boolean responses: @@ -1995,12 +1995,15 @@ paths: schema: type: object properties: - exists_openai_resource_path: + ai_provider: + type: string + exists_ai_resource: type: boolean code_completion_enabled: type: boolean required: - - exists_openai_resource_path + - ai_provider + - exists_ai_resource - code_completion_enabled /w/{workspace}/workspaces/edit_error_handler: @@ -10194,6 +10197,16 @@ components: schemas: $ref: "../../openflow.openapi.yaml#/components/schemas" + AiResource: + type: object + properties: + path: + type: string + provider: + type: string + required: + - path + - provider Script: type: object properties: @@ -11012,7 +11025,7 @@ components: - "jobs.disapproval" - "jobs.delete" - "account.delete" - - "openai.request" + - "ai.request" - "resources.create" - "resources.update" - "resources.delete" diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs new file mode 100644 index 0000000000..32c1f4e4dc --- /dev/null +++ b/backend/windmill-api/src/ai.rs @@ -0,0 +1,507 @@ +use crate::{ + db::{ApiAuthed, DB}, + variables::decrypt, +}; +use anthropic::AnthropicCache; +use axum::{ + body::Bytes, + extract::{Path, Query}, + response::IntoResponse, + routing::post, + Extension, Router, +}; +use lazy_static::lazy_static; +use mistral::MistralCache; +use openai::OpenaiCache; +use quick_cache::sync::Cache; +use reqwest::{Client, RequestBuilder}; +use serde::{Deserialize, Deserializer}; +use windmill_audit::audit_ee::audit_log; +use windmill_audit::ActionKind; +use windmill_common::error::{to_anyhow, Result}; +use windmill_common::variables::build_crypt; + +use windmill_common::error::Error; + +use serde_json::value::{RawValue, Value}; +use std::collections::HashMap; + +lazy_static::lazy_static! { + static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new() + .timeout(std::time::Duration::from_secs(60 * 5)) + .user_agent("windmill/beta") + .build().unwrap(); +} + +trait AiRequest { + fn prepare_request(self, path: &str, body: Bytes) -> Result; +} + +mod openai { + use super::*; + + use super::{get_variable_or_self, KeyCache}; + + const API_VERSION: &str = "2023-05-15"; + + #[derive(Deserialize, Debug)] + struct OpenaiResource { + api_key: String, + organization_id: Option, + } + + #[derive(Deserialize, Debug)] + struct OpenaiClientCredentialsOauthResource { + client_id: String, + client_secret: String, + token_url: String, + user: Option, + } + + #[derive(Deserialize, Debug)] + #[serde(untagged, rename_all = "snake_case")] + enum OpenaiConfig { + Resource(OpenaiResource), + ClientCredentialsOauthResource(OpenaiClientCredentialsOauthResource), + } + + lazy_static::lazy_static! { + pub static ref OPENAI_AZURE_BASE_PATH: Option = std::env::var("OPENAI_AZURE_BASE_PATH").ok(); + } + + #[derive(Deserialize, Debug)] + struct OpenaiCredentials { + access_token: String, + } + + #[derive(Clone, Debug, Deserialize)] + pub struct OpenaiCache { + api_key: String, + organization_id: Option, + azure_base_path: Option, + user: Option, + } + + impl OpenaiCache { + pub fn new( + api_key: String, + organization_id: Option, + azure_base_path: Option, + user: Option, + ) -> Self { + Self { api_key, organization_id, azure_base_path, user } + } + } + + const BASE_URL: &str = "https://api.openai.com/v1"; + impl AiRequest for OpenaiCache { + fn prepare_request(self, openai_path: &str, mut body: Bytes) -> Result { + let OpenaiCache { api_key, azure_base_path, organization_id, user } = self; + if user.is_some() { + tracing::debug!("Adding user to request body"); + let mut json_body: HashMap> = serde_json::from_slice(&body) + .map_err(|e| { + Error::InternalErr(format!("Failed to parse request body: {}", e)) + })?; + + let user_json_string = serde_json::Value::String(user.unwrap()).to_string(); // makes sure to escape characters + + json_body.insert( + "user".to_string(), + RawValue::from_string(user_json_string) + .map_err(|e| Error::InternalErr(format!("Failed to parse user: {}", e)))?, + ); + + body = serde_json::to_vec(&json_body) + .map_err(|e| { + Error::InternalErr(format!("Failed to reserialize request body: {}", e)) + })? + .into(); + } + + let base_url = if let Some(base_url) = azure_base_path { + base_url + } else { + BASE_URL.to_string() + }; + let url = format!("{}/{}", base_url, openai_path); + let mut request = HTTP_CLIENT + .post(url) + .header("content-type", "application/json") + .body(body); + + if base_url != BASE_URL { + request = request + .header("api-key", api_key) + .query(&[("api-version", API_VERSION)]) + } else { + request = request.header("authorization", format!("Bearer {}", api_key)) + } + + if let Some(org_id) = organization_id { + request = request.header("OpenAI-Organization", org_id); + } + + Ok(request) + } + } + + async fn get_openai_key_using_credentials_flow( + mut resource: OpenaiClientCredentialsOauthResource, + db: &DB, + w_id: &str, + ) -> Result { + resource.client_id = get_variable_or_self(resource.client_id, db, w_id).await?; + resource.client_secret = get_variable_or_self(resource.client_secret, db, w_id).await?; + resource.token_url = get_variable_or_self(resource.token_url, db, w_id).await?; + let mut params = HashMap::new(); + params.insert("grant_type", "client_credentials"); + let response = HTTP_CLIENT + .post(resource.token_url) + .form(¶ms) + .basic_auth(resource.client_id, Some(resource.client_secret)) + .send() + .await + .map_err(|err| { + Error::InternalErr(format!( + "Failed to get OpenAI credentials using credentials flow: {}", + err + )) + })?; + let response = response.json::().await.map_err(|err| { + Error::InternalErr(format!( + "Failed to parse OpenAI credentials from credentials flow: {}", + err + )) + })?; + Ok(response.access_token) + } + + pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result { + let config = serde_json::from_value(resource) + .map_err(|e| Error::InternalErr(format!("validating openai resource {e:#}")))?; + + let mut user = None::; + let mut resource = match config { + OpenaiConfig::Resource(resource) => { + tracing::debug!("Getting OpenAI key from static resource"); + resource + } + OpenaiConfig::ClientCredentialsOauthResource(resource) => { + tracing::debug!("Getting OpenAI key with client credentials flow"); + user = resource.user.clone(); + let token = get_openai_key_using_credentials_flow(resource, db, w_id).await?; + OpenaiResource { api_key: token, organization_id: None } + } + }; + + resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?; + + if let Some(organization_id) = resource.organization_id { + resource.organization_id = Some(get_variable_or_self(organization_id, db, w_id).await?); + } + + if user.is_some() { + user = Some(get_variable_or_self(user.unwrap(), db, w_id).await?); + } + + let azure_base_path = sqlx::query_scalar!( + "SELECT value + FROM global_settings + WHERE name = 'openai_azure_base_path'", + ) + .fetch_optional(db) + .await?; + + let azure_base_path = if let Some(azure_base_path) = azure_base_path { + Some( + serde_json::from_value::(azure_base_path).map_err(|e| { + Error::InternalErr(format!("validating openai azure base path {e:#}")) + })?, + ) + } else { + OPENAI_AZURE_BASE_PATH.clone() + }; + + let workspace_cache = OpenaiCache::new( + resource.api_key.clone(), + resource.organization_id.clone(), + azure_base_path.clone(), + user.clone(), + ); + Ok(KeyCache::Openai(workspace_cache)) + } +} + +mod anthropic { + + use super::*; + + #[derive(Clone, Deserialize, Debug)] + pub struct AnthropicCache { + #[serde(rename = "apiKey")] + pub api_key: String, + } + + const API_VERSION: &str = "2023-06-01"; + + impl AnthropicCache { + pub fn new(api_key: String) -> Self { + Self { api_key } + } + } + + const BASE_URL: &str = "https://api.anthropic.com"; + impl AiRequest for AnthropicCache { + fn prepare_request(self, anthropic_path: &str, body: Bytes) -> Result { + let AnthropicCache { api_key } = self; + let url = format!("{}/{}", BASE_URL, anthropic_path); + let request = HTTP_CLIENT + .post(url) + .header("x-api-key", api_key) + .header("anthropic-version", API_VERSION) + .header("content-type", "application/json") + .body(body); + Ok(request) + } + } + + pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result { + let mut resource: AnthropicCache = serde_json::from_value(resource) + .map_err(|e| Error::InternalErr(format!("validating anthropic resource {e:#}")))?; + resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?; + let workspace_cache = AnthropicCache::new(resource.api_key); + Ok(KeyCache::Anthropic(workspace_cache)) + } +} + +mod mistral { + use super::*; + #[derive(Deserialize, Clone, Debug)] + pub struct MistralCache { + #[serde(rename = "apiKey")] + pub api_key: String, + } + + impl MistralCache { + pub fn new(api_key: String) -> Self { + Self { api_key } + } + } + + const BASE_URL: &str = "https://api.mistral.ai"; + impl AiRequest for MistralCache { + fn prepare_request(self, mistral_path: &str, body: Bytes) -> Result { + let MistralCache { api_key } = self; + + let url = format!("{}/{}", BASE_URL, mistral_path); + let request = HTTP_CLIENT + .post(url) + .header("content-type", "application/json") + .header("Accept", "application/json") + .header("authorization", format!("Bearer {}", api_key)) + .body(body); + Ok(request) + } + } + + pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result { + let mut resource: MistralCache = serde_json::from_value(resource) + .map_err(|e| Error::InternalErr(format!("validating mistral resource {e:#}")))?; + resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?; + + let workspace_cache = MistralCache::new(resource.api_key); + Ok(KeyCache::Mistral(workspace_cache)) + } +} + +#[derive(Clone, Debug)] +pub enum KeyCache { + Openai(OpenaiCache), + Anthropic(AnthropicCache), + Mistral(MistralCache), +} + +#[derive(Clone, Debug)] +pub struct AiCache { + pub path: String, + pub cached_key: KeyCache, + pub expires_at: std::time::Instant, +} + +impl AiCache { + pub fn new(path: String, cached_key: KeyCache) -> Self { + Self { + path, + cached_key, + expires_at: std::time::Instant::now() + std::time::Duration::from_secs(60), + } + } + fn is_expired(&self) -> bool { + self.expires_at < std::time::Instant::now() + } +} + +lazy_static! { + pub static ref AI_KEY_CACHE: Cache = Cache::new(500); +} + +struct Variable { + value: String, + is_secret: bool, +} + +#[derive(Deserialize, Debug)] +struct ProxyQueryParams { + no_cache: Option, +} + +async fn get_variable_or_self(path: String, db: &DB, w_id: &str) -> Result { + if !path.starts_with("$var:") { + return Ok(path); + } + let path = path.strip_prefix("$var:").unwrap().to_string(); + let mut variable = sqlx::query_as!( + Variable, + "SELECT value, is_secret + FROM variable + WHERE path = $1 AND workspace_id = $2", + &path, + &w_id + ) + .fetch_one(db) + .await?; + if variable.is_secret { + let mc = build_crypt(db, w_id).await?; + variable.value = decrypt(&mc, variable.value)?; + } + Ok(variable.value) +} + +#[derive(Deserialize, Debug)] +pub struct AiResource { + pub path: String, + #[serde(deserialize_with = "check_if_valid_ai_provider")] + pub provider: String, +} + +fn check_if_valid_ai_provider<'de, D>(provider: D) -> std::result::Result +where + D: Deserializer<'de>, +{ + let provider = String::deserialize(provider)?; + match provider.as_str() { + "anthropic" | "openai" | "mistral" => Ok(provider), + _ => Err(serde::de::Error::custom( + "Only the following Ai providers are supported: openai, anthropic and mistral" + .to_string(), + )), + } +} + +pub fn workspaced_service() -> Router { + let router = Router::new().route("/proxy/*ai", post(proxy)); + + router +} + +async fn proxy( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, ai_path)): Path<(String, String)>, + Query(query_params): Query, + body: Bytes, +) -> impl IntoResponse { + let workspace_cache = AI_KEY_CACHE.get(&w_id); + let ai_cache = match workspace_cache { + Some(cache) if !cache.is_expired() && !query_params.no_cache.unwrap_or(false) => { + cache.cached_key + } + _ => { + let ai_resource = sqlx::query_scalar!( + "SELECT ai_resource FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; + + if ai_resource.is_none() { + return Err(Error::InternalErr("AI resource not configured".to_string())); + } + + let ai_resource = serde_json::from_value::(ai_resource.unwrap()) + .map_err(|e| Error::BadRequest(e.to_string()))?; + let ai_resource_path = ai_resource.path; + + let resource = sqlx::query_scalar!( + "SELECT value + FROM resource + WHERE path = $1 AND workspace_id = $2", + &ai_resource_path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::InternalErr(format!( + "Could not find the {} resource at path {ai_resource_path}, update the resource path in the workspace settings", ai_resource.provider + )) + })?; + + if resource.is_none() { + return Err(Error::InternalErr(format!( + "{} resource missing value", + ai_resource.provider + ))); + } + + let resource = resource.unwrap(); + + let ai_cache = match ai_resource.provider.as_str() { + "openai" => openai::get_cached_value(&db, &w_id, resource).await, + "anthropic" => anthropic::get_cached_value(&db, &w_id, resource).await, + "mistral" => mistral::get_cached_value(&db, &w_id, resource).await, + provider => { + return Err(Error::BadRequest(format!("{} is not supported", provider))) + } + }; + let ai_cache = ai_cache?; + AI_KEY_CACHE.insert( + w_id.clone(), + AiCache::new(ai_resource_path, ai_cache.clone()), + ); + ai_cache + } + }; + let (path, request) = match ai_cache { + KeyCache::Openai(cached) => ("openai_path", cached.prepare_request(&ai_path, body)), + KeyCache::Anthropic(cached) => ("anthropic_path", cached.prepare_request(&ai_path, body)), + KeyCache::Mistral(cached) => ("mistral_path", cached.prepare_request(&ai_path, body)), + }; + + let response = request?.send().await.map_err(to_anyhow)?; + + let mut tx = db.begin().await?; + + audit_log( + &mut *tx, + &authed, + "ai.request", + ActionKind::Execute, + &w_id, + Some(&authed.email), + Some([(path, &format!("{:?}", ai_path)[..])].into()), + ) + .await?; + tx.commit().await?; + + if response.error_for_status_ref().is_err() { + let err_msg = response.text().await.unwrap_or("".to_string()); + return Err(Error::AiError(err_msg)); + } + + let status_code = response.status(); + let headers = response.headers().clone(); + let stream = response.bytes_stream(); + Ok((status_code, headers, axum::body::Body::from_stream(stream))) +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index fe74584d1b..aa6e6955e6 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -62,13 +62,14 @@ mod http_triggers; mod indexer_ee; mod inputs; mod integration; +mod ai; + #[cfg(feature = "parquet")] mod job_helpers_ee; pub mod job_metrics; pub mod jobs; pub mod oauth2_ee; mod oidc_ee; -mod openai; mod raw_apps; mod resources; mod saml_ee; @@ -94,6 +95,7 @@ mod workspaces_ee; pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB + lazy_static::lazy_static! { pub static ref REQUEST_SIZE_LIMIT: Arc> = Arc::new(RwLock::new(DEFAULT_BODY_LIMIT)); @@ -278,7 +280,7 @@ pub async fn run_server( .nest("/job_helpers", job_helpers_service) .nest("/jobs", jobs::workspaced_service()) .nest("/oauth", oauth2_ee::workspaced_service()) - .nest("/openai", openai::workspaced_service()) + .nest("/ai", ai::workspaced_service()) .nest("/raw_apps", raw_apps::workspaced_service()) .nest("/resources", resources::workspaced_service()) .nest("/schedules", schedule::workspaced_service()) diff --git a/backend/windmill-api/src/openai.rs b/backend/windmill-api/src/openai.rs deleted file mode 100644 index 0794dadaa5..0000000000 --- a/backend/windmill-api/src/openai.rs +++ /dev/null @@ -1,344 +0,0 @@ -use std::collections::HashMap; - -use crate::db::{ApiAuthed, DB}; - -use axum::{ - body::Bytes, - extract::{Extension, Path, Query}, - response::IntoResponse, - routing::post, - Router, -}; -use quick_cache::sync::Cache; -use reqwest::Client; -use serde_json::value::RawValue; -use windmill_audit::audit_ee::audit_log; -use windmill_audit::ActionKind; -use windmill_common::{ - error::{to_anyhow, Error}, - variables::build_crypt, -}; - -use crate::variables::decrypt; -use serde::Deserialize; - -lazy_static::lazy_static! { - static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new() - .timeout(std::time::Duration::from_secs(60 * 5)) - .user_agent("windmill/beta") - .build().unwrap(); -} - -pub fn workspaced_service() -> Router { - let router = Router::new().route("/proxy/*openai_path", post(proxy)); - - router -} - -#[derive(Deserialize)] -struct OpenaiResource { - api_key: String, - organization_id: Option, -} - -#[derive(Deserialize)] -struct OpenaiClientCredentialsOauthResource { - client_id: String, - client_secret: String, - token_url: String, - user: Option, -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum OpenaiConfig { - Resource(OpenaiResource), - ClientCredentialsOauthResource(OpenaiClientCredentialsOauthResource), -} - -struct Variable { - value: String, - is_secret: bool, -} -async fn get_variable_or_self(path: String, db: &DB, w_id: &String) -> Result { - if !path.starts_with("$var:") { - return Ok(path); - } - let path = path.strip_prefix("$var:").unwrap().to_string(); - let mut variable = sqlx::query_as!( - Variable, - "SELECT value, is_secret - FROM variable - WHERE path = $1 AND workspace_id = $2", - &path, - &w_id - ) - .fetch_one(db) - .await?; - if variable.is_secret { - let mc = build_crypt(&db, &w_id).await?; - variable.value = decrypt(&mc, variable.value)?; - } - Ok(variable.value) -} - -lazy_static::lazy_static! { - pub static ref OPENAI_AZURE_BASE_PATH: Option = std::env::var("OPENAI_AZURE_BASE_PATH").ok(); - static ref OPENAI_KEY_CACHE: Cache = Cache::new(500); -} - -#[derive(Deserialize)] -struct OpenaiCredentials { - access_token: String, -} -async fn get_openai_key_using_credentials_flow( - mut resource: OpenaiClientCredentialsOauthResource, - db: &DB, - w_id: &String, -) -> Result { - resource.client_id = get_variable_or_self(resource.client_id, &db, &w_id).await?; - resource.client_secret = get_variable_or_self(resource.client_secret, &db, &w_id).await?; - resource.token_url = get_variable_or_self(resource.token_url, &db, &w_id).await?; - let mut params = HashMap::new(); - params.insert("grant_type", "client_credentials"); - let response = HTTP_CLIENT - .post(resource.token_url) - .form(¶ms) - .basic_auth(resource.client_id, Some(resource.client_secret)) - .send() - .await - .map_err(|err| { - Error::InternalErr(format!( - "Failed to get OpenAI credentials using credentials flow: {}", - err - )) - })?; - let response = response.json::().await.map_err(|err| { - Error::InternalErr(format!( - "Failed to parse OpenAI credentials from credentials flow: {}", - err - )) - })?; - Ok(response.access_token) -} - -#[derive(Clone)] -struct OpenaiKeyCache { - api_key: String, - organization_id: Option, - azure_base_path: Option, - user: Option, - expires_at: std::time::Instant, -} - -impl OpenaiKeyCache { - fn new( - api_key: String, - organization_id: Option, - azure_base_path: Option, - expires_at: std::time::Instant, - user: Option, - ) -> Self { - Self { api_key, organization_id, azure_base_path, expires_at, user } - } - fn is_expired(&self) -> bool { - self.expires_at < std::time::Instant::now() - } -} - -#[derive(Deserialize)] -struct ProxyQueryParams { - no_cache: Option, -} -async fn proxy( - authed: ApiAuthed, - Extension(db): Extension, - Path((w_id, openai_path)): Path<(String, String)>, - Query(query_params): Query, - mut body: Bytes, -) -> impl IntoResponse { - let workspace_cache = OPENAI_KEY_CACHE.get(&w_id); - let (api_key, organization_id, azure_base_path, user) = if query_params - .no_cache - .unwrap_or(false) - || workspace_cache.is_none() - || workspace_cache.clone().unwrap().is_expired() - { - let openai_resource_path = sqlx::query_scalar!( - "SELECT openai_resource_path FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; - - if openai_resource_path.is_none() { - return Err(Error::InternalErr( - "OpenAI resource not configured".to_string(), - )); - } - - let openai_resource_path = openai_resource_path.unwrap(); - - let resource = sqlx::query_scalar!( - "SELECT value - FROM resource - WHERE path = $1 AND workspace_id = $2", - &openai_resource_path, - &w_id - ) - .fetch_optional(&db) - .await? - .ok_or_else(|| { - Error::InternalErr(format!( - "Could not find the OpenAI resource at path {openai_resource_path}, update the resource path in the workspace settings" - )) - })?; - - if resource.is_none() { - return Err(Error::InternalErr( - "OpenAI resource missing value".to_string(), - )); - } - - let config: OpenaiConfig = serde_json::from_value(resource.unwrap()) - .map_err(|e| Error::InternalErr(format!("validating openai resource {e:#}")))?; - - let mut user = None::; - let mut resource = match config { - OpenaiConfig::Resource(resource) => { - tracing::debug!("Getting OpenAI key from static resource"); - resource - } - OpenaiConfig::ClientCredentialsOauthResource(resource) => { - tracing::debug!("Getting OpenAI key with client credentials flow"); - user = resource.user.clone(); - let token = get_openai_key_using_credentials_flow(resource, &db, &w_id).await?; - OpenaiResource { api_key: token, organization_id: None } - } - }; - - resource.api_key = get_variable_or_self(resource.api_key, &db, &w_id).await?; - - if resource.organization_id.is_some() { - resource.organization_id = - Some(get_variable_or_self(resource.organization_id.unwrap(), &db, &w_id).await?); - } - - if user.is_some() { - user = Some(get_variable_or_self(user.unwrap(), &db, &w_id).await?); - } - - let expires_at = std::time::Instant::now() + std::time::Duration::from_secs(60); - - let azure_base_path = sqlx::query_scalar!( - "SELECT value - FROM global_settings - WHERE name = 'openai_azure_base_path'", - ) - .fetch_optional(&db) - .await?; - - let azure_base_path = if let Some(azure_base_path) = azure_base_path { - Some( - serde_json::from_value::(azure_base_path).map_err(|e| { - Error::InternalErr(format!("validating openai azure base path {e:#}")) - })?, - ) - } else { - OPENAI_AZURE_BASE_PATH.clone() - }; - - let workspace_cache = OpenaiKeyCache::new( - resource.api_key.clone(), - resource.organization_id.clone(), - azure_base_path.clone(), - expires_at, - user.clone(), - ); - OPENAI_KEY_CACHE.insert(w_id.clone(), workspace_cache); - ( - resource.api_key, - resource.organization_id, - azure_base_path, - user, - ) - } else { - tracing::debug!("Using cached OpenAI key"); - let workspace_cache = workspace_cache.unwrap(); - ( - workspace_cache.api_key.clone(), - workspace_cache.organization_id.clone(), - workspace_cache.azure_base_path.clone(), - workspace_cache.user.clone(), - ) - }; - - if user.is_some() { - tracing::debug!("Adding user to request body"); - let mut json_body: HashMap> = serde_json::from_slice(&body) - .map_err(|e| Error::InternalErr(format!("Failed to parse request body: {}", e)))?; - - let user_json_string = serde_json::Value::String(user.unwrap()).to_string(); // makes sure to escape characters - - json_body.insert( - "user".to_string(), - RawValue::from_string(user_json_string) - .map_err(|e| Error::InternalErr(format!("Failed to parse user: {}", e)))?, - ); - - body = serde_json::to_vec(&json_body) - .map_err(|e| Error::InternalErr(format!("Failed to reserialize request body: {}", e)))? - .into(); - } - - let base_url = if let Some(base_url) = azure_base_path { - base_url - } else { - "https://api.openai.com/v1".to_string() - }; - - let url = format!("{}/{}", base_url, openai_path); - let mut request = HTTP_CLIENT - .post(url) - .header("content-type", "application/json") - .body(body); - - if base_url != "https://api.openai.com/v1" { - request = request - .header("api-key", api_key) - .query(&[("api-version", "2023-05-15")]) - } else { - request = request.header("authorization", format!("Bearer {}", api_key)) - } - - if let Some(org_id) = organization_id { - request = request.header("OpenAI-Organization", org_id); - } - - let response = request.send().await.map_err(to_anyhow)?; - - let mut tx = db.begin().await?; - audit_log( - &mut *tx, - &authed, - "openai.request", - ActionKind::Execute, - &w_id, - Some(&authed.email), - Some([("openai_path", &format!("{:?}", openai_path)[..])].into()), - ) - .await?; - tx.commit().await?; - - if response.error_for_status_ref().is_err() { - return Err(Error::OpenAIError( - response.text().await.unwrap_or("".to_string()), - )); - } - - let status_code = response.status(); - let headers = response.headers().clone(); - let stream = response.bytes_stream(); - - Ok((status_code, headers, axum::body::Body::from_stream(stream))) -} diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 018ac8de5f..ddd227ecbb 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; +use crate::ai::{AiResource, AI_KEY_CACHE}; use crate::db::ApiAuthed; use crate::users_ee::send_email_if_possible; use crate::utils::get_instance_username_or_create_pending; @@ -168,7 +169,7 @@ pub struct WorkspaceSettings { pub plan: Option, pub webhook: Option, pub deploy_to: Option, - pub openai_resource_path: Option, + pub ai_resource: Option, pub code_completion_enabled: bool, pub error_handler: Option, pub error_handler_extra_args: Option, @@ -234,7 +235,7 @@ struct EditWebhook { #[derive(Deserialize)] struct EditCopilotConfig { - openai_resource_path: Option, + ai_resource: Option, code_completion_enabled: bool, } @@ -645,23 +646,33 @@ async fn edit_copilot_config( let mut tx = db.begin().await?; - if let Some(openai_resource_path) = &eo.openai_resource_path { + if let Some(ai_resource) = &eo.ai_resource { + let path = serde_json::from_value::(ai_resource.clone()) + .map_err(|e| Error::BadRequest(e.to_string()))? + .path; sqlx::query!( - "UPDATE workspace_settings SET openai_resource_path = $1, code_completion_enabled = $2 WHERE workspace_id = $3", - openai_resource_path, + "UPDATE workspace_settings SET ai_resource = $1, code_completion_enabled = $2 WHERE workspace_id = $3", + ai_resource, eo.code_completion_enabled, &w_id ) .execute(&mut *tx) .await?; + + if let Some(cached) = AI_KEY_CACHE.get(&w_id) { + if cached.path != path { + AI_KEY_CACHE.remove(&w_id); + } + } } else { sqlx::query!( - "UPDATE workspace_settings SET openai_resource_path = NULL, code_completion_enabled = $1 WHERE workspace_id = $2", + "UPDATE workspace_settings SET ai_resource = NULL, code_completion_enabled = $1 WHERE workspace_id = $2", eo.code_completion_enabled, &w_id, ) .execute(&mut *tx) .await?; + AI_KEY_CACHE.remove(&w_id); } audit_log( &mut *tx, @@ -672,10 +683,7 @@ async fn edit_copilot_config( Some(&authed.email), Some( [ - ( - "openai_resource_path", - &format!("{:?}", eo.openai_resource_path)[..], - ), + ("ai_resource", &format!("{:?}", eo.ai_resource)[..]), ( "code_completion_enabled", &format!("{:?}", eo.code_completion_enabled)[..], @@ -692,7 +700,8 @@ async fn edit_copilot_config( #[derive(Serialize)] struct CopilotInfo { - pub exists_openai_resource_path: bool, + pub ai_provider: String, + pub exists_ai_resource: bool, pub code_completion_enabled: bool, } async fn get_copilot_info( @@ -701,16 +710,32 @@ async fn get_copilot_info( ) -> JsonResult { let mut tx = db.begin().await?; let record = sqlx::query!( - "SELECT openai_resource_path, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1", + "SELECT ai_resource, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1", &w_id ) .fetch_one(&mut *tx) .await - .map_err(|e| Error::InternalErr(format!("getting openai_resource_path and code_completion_enabled: {e:#}")))?; + .map_err(|e| Error::InternalErr(format!("getting ai_resource and code_completion_enabled: {e:#}")))?; tx.commit().await?; + let (ai_provider, exists_ai_resource) = if let Some(ai_resource) = record.ai_resource { + let ai_resource = serde_json::from_value::(ai_resource); + let exist = ai_resource.is_ok(); + ( + if exist { + ai_resource.unwrap().provider + } else { + "".to_string() + }, + exist, + ) + } else { + ("".to_string(), false) + }; + Ok(Json(CopilotInfo { - exists_openai_resource_path: record.openai_resource_path.is_some(), + ai_provider, + exists_ai_resource, code_completion_enabled: record.code_completion_enabled, })) } @@ -2223,7 +2248,7 @@ struct SimplifiedSettings { error_handler: Option, error_handler_extra_args: Option, error_handler_muted_on_cancel: bool, - openai_resource_path: Option, + ai_resource: Option, code_completion_enabled: bool, large_file_storage: Option, git_sync: Option, @@ -2588,7 +2613,7 @@ async fn tarball_workspace( webhook, deploy_to, error_handler, - openai_resource_path, + ai_resource, code_completion_enabled, error_handler_extra_args, error_handler_muted_on_cancel, diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 8ac8537c11..ac57b64914 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -59,7 +59,7 @@ pub enum Error { #[error("Error: {0:#?}")] JsonErr(serde_json::Value), #[error("{0}")] - OpenAIError(String), + AiError(String), #[error("{0}")] AlreadyCompleted(String), } @@ -90,7 +90,7 @@ impl IntoResponse for Error { Self::RequireAdmin(_) => axum::http::StatusCode::FORBIDDEN, Self::SqlErr(_) | Self::BadRequest(_) - | Self::OpenAIError(_) + | Self::AiError(_) | Self::QuotaExceeded(_) => axum::http::StatusCode::BAD_REQUEST, _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR, }; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 2b1bd0e614..4b63b77cf2 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,6 +9,7 @@ "version": "1.423.2", "license": "AGPL-3.0", "dependencies": { + "@anthropic-ai/sdk": "^0.32.1", "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~8.0.2", "@codingame/monaco-vscode-files-service-override": "~8.0.2", @@ -21,6 +22,7 @@ "@codingame/monaco-vscode-standalone-typescript-language-features": "~8.0.2", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", + "@mistralai/mistralai": "^1.3.0", "@popperjs/core": "^2.11.6", "@redocly/json-to-json-schema": "^0.0.1", "@tanstack/svelte-table": "^8.9.9", @@ -175,6 +177,30 @@ "node": ">=6.0.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.32.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.32.1.tgz", + "integrity": "sha512-U9JwTrDvdQ9iWuABVsMLj8nJVwAyQz6QXvgLsVhryhCEPkLsbcP/MXxm+jYcAwLoV8ESbaTTjnD4kuAFa+Hyjg==", + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { + "version": "18.19.64", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.64.tgz", + "integrity": "sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "11.6.1", "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.6.1.tgz", @@ -3659,6 +3685,14 @@ "resolved": "https://registry.npmjs.org/@mapbox/unitbezier/-/unitbezier-0.0.0.tgz", "integrity": "sha512-HPnRdYO0WjFjRTSwO3frz1wKaU649OBFPX3Zo/2WZvuRi6zMiRGui8SnPQiQABgqCf8YikDe5t3HViTVw1WUzA==" }, + "node_modules/@mistralai/mistralai": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-1.3.0.tgz", + "integrity": "sha512-G5DPCSC8sEhG3LUEZDYLD7qEZWDuZXgaX3IcoC3a/ydm9jFuh2pRZtknsgMx2sU8d7kxRuxblY3fPH5C38wnhQ==", + "peerDependencies": { + "zod": ">= 3" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -13738,6 +13772,16 @@ "integrity": "sha512-A7AMeJfGefk317I/3tBoUYRcDcNavKEkpiPN/nQsBz/viI2GvT7BtrqdPD6rGqBFN8Ax7v4obf+Cl32JF9DDVw==", "dev": true }, + "node_modules/zod": { + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/zstddec": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/zstddec/-/zstddec-0.1.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index 4465bf84b5..5b521f7d77 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -82,6 +82,7 @@ }, "type": "module", "dependencies": { + "@anthropic-ai/sdk": "^0.32.1", "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~8.0.2", "@codingame/monaco-vscode-files-service-override": "~8.0.2", @@ -94,6 +95,7 @@ "@codingame/monaco-vscode-standalone-typescript-language-features": "~8.0.2", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", + "@mistralai/mistralai": "^1.3.0", "@popperjs/core": "^2.11.6", "@redocly/json-to-json-schema": "^0.0.1", "@tanstack/svelte-table": "^8.9.9", diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index 5438c7edd4..90659e3103 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -39,7 +39,6 @@ import { CornerDownLeft, Play } from 'lucide-svelte' import Toggle from './Toggle.svelte' import { setLicense } from '$lib/enterpriseUtils' - import { workspacedOpenai } from './copilot/lib' import type { FlowCopilotContext, FlowCopilotModule } from './copilot/flow' import { pickScript } from './flows/flowStateUtils' import { @@ -49,6 +48,7 @@ } from '$lib/relative_imports' import Tooltip from './Tooltip.svelte' import type { ScheduleTrigger, TriggerContext } from './triggers' + import { initAllAiWorkspace } from './copilot/lib' import type { FlowPropPickerConfig, PropPickerContext } from './prop_picker' import type { PickableProperties } from './flows/previousResults' $: token = $page.url.searchParams.get('wm_token') ?? undefined @@ -110,12 +110,13 @@ async function setCopilotInfo() { if (workspace) { - workspacedOpenai.init(workspace, token) + initAllAiWorkspace(workspace) try { copilotInfo.set(await WorkspaceService.getCopilotInfo({ workspace })) } catch (err) { copilotInfo.set({ - exists_openai_resource_path: false, + ai_provider: '', + exists_ai_resource: false, code_completion_enabled: false }) diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 091a6f927d..035a5afca3 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -171,6 +171,7 @@ import { initVim } from './monaco_keybindings' import { buildWorkerDefinition } from '$lib/monaco_workers/build_workers' import { parseTypescriptDeps } from '$lib/relative_imports' + import type { AiProviderTypes } from './copilot/lib' // import EditorTheme from './EditorTheme.svelte' @@ -593,11 +594,13 @@ token.onCancellationRequested(() => { abortController?.abort() }) + const aiProvider = $copilotInfo.ai_provider const insertText = await editorCodeCompletion( textUntilPosition, textAfterPosition, lang, - abortController + abortController, + aiProvider as AiProviderTypes ) if (insertText) { items = [ @@ -624,7 +627,7 @@ ) } - $: $copilotInfo.exists_openai_resource_path && + $: $copilotInfo.exists_ai_resource && $copilotInfo.code_completion_enabled && $codeCompletionSessionEnabled && initialized && diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index f51b7e8bc8..c90525db4b 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -85,6 +85,7 @@ import type { FlowBuilderWhitelabelCustomUi } from './custom_ui' import FlowYamlEditor from './flows/header/FlowYamlEditor.svelte' import { type TriggerContext, type ScheduleTrigger } from './triggers' + import type { AiProviderTypes } from './copilot/lib' export let initialPath: string = '' export let pathStoreInit: string | undefined = undefined @@ -789,6 +790,7 @@ try { push(history, $flowStore) let module = stepOnly ? $copilotModulesStore[0] : $copilotModulesStore[idx] + const aiProvider = $copilotInfo.ai_provider as AiProviderTypes copilotLoading = true copilotStatus = "Generating code for step '" + module.id + "'..." @@ -918,7 +920,8 @@ }) : undefined, isFirstInLoop, - abortController + abortController, + aiProvider ) unsubscribe() } @@ -934,7 +937,7 @@ pastModule.value.type === 'script') ) { const stepSchema: Schema = JSON.parse(JSON.stringify($flowStateStore[module.id].schema)) // deep copy - if (isHubStep && pastModule !== undefined && $copilotInfo.exists_openai_resource_path) { + if (isHubStep && pastModule !== undefined && $copilotInfo.exists_ai_resource) { // ask AI to set step inputs abortController = new AbortController() const { inputs, allExprs } = await glueCopilot( @@ -944,7 +947,8 @@ value: RawScript | PathScript }, isFirstInLoop, - abortController + abortController, + aiProvider ) // create flow inputs used by AI for autocompletion @@ -992,11 +996,7 @@ $shouldUpdatePropertyType[key] = 'javascript' }) } else { - if ( - isHubStep && - pastModule !== undefined && - !$copilotInfo.exists_openai_resource_path - ) { + if (isHubStep && pastModule !== undefined && !$copilotInfo.exists_ai_resource) { sendUserToast( 'For better input generation, enable Windmill AI in the workspace settings', true diff --git a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte index 0610d7f571..40dea32eb0 100644 --- a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte +++ b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte @@ -221,7 +221,7 @@ JOBS_DISAPPROVAL: 'jobs.disapproval', JOBS_DELETE: 'jobs.delete', ACCOUNT_DELETE: 'account.delete', - OPENAI_REQUEST: 'openai.request', + AI_REQUEST: 'ai.request', RESOURCES_CREATE: 'resources.create', RESOURCES_UPDATE: 'resources.update', RESOURCES_DELETE: 'resources.delete', diff --git a/frontend/src/lib/components/copilot/CodeCompletionStatus.svelte b/frontend/src/lib/components/copilot/CodeCompletionStatus.svelte index 75d7a0decf..8a77a8df9f 100644 --- a/frontend/src/lib/components/copilot/CodeCompletionStatus.svelte +++ b/frontend/src/lib/components/copilot/CodeCompletionStatus.svelte @@ -12,7 +12,7 @@ } -{#if $copilotInfo.exists_openai_resource_path && $copilotInfo.code_completion_enabled} +{#if $copilotInfo.exists_ai_resource && $copilotInfo.code_completion_enabled} import { ExternalLink, Wand2 } from 'lucide-svelte' import Button from '../common/button/Button.svelte' - import { getNonStreamingCompletion } from './lib' + import { getNonStreamingCompletion, type AiProviderTypes } from './lib' import Popup from '../common/popup/Popup.svelte' import { sendUserToast } from '$lib/toast' import { copilotInfo } from '$lib/stores' + import { base } from '$lib/base' + import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' export let schedule: string @@ -14,31 +16,35 @@ let genLoading = false let abortController = new AbortController() $: instructionsField && setTimeout(() => instructionsField?.focus(), 100) + const SYSTEM = "You are a helpful assitant for creating CRON schedules. The structure is 'second minute hour dayOfMonth month dayOfWeek'. Weekdays are Sunday (1), Monday (2), Tuesday (3), Wednesday (4), Thursday (5), Friday (6), Saturday (7). You only return the CRON string without any wrapping characters. If it is invalid, you will return an error message preceeded by 'ERROR:'." const USER = 'CRON schedule instructions: {instructions}' async function generateCron() { genLoading = true abortController = new AbortController() + const aiProvider = $copilotInfo.ai_provider try { + const messages: ChatCompletionMessageParam[] = [ + { + role: 'system', + content: SYSTEM + }, + { + role: 'user', + content: USER.replace('{instructions}', instructions) + } + ] + const response = await getNonStreamingCompletion( - [ - { - role: 'system', - content: SYSTEM - }, - { - role: 'user', - content: USER.replace('{instructions}', instructions) - } - ], - abortController + messages, + abortController, + aiProvider as AiProviderTypes ) if (response.startsWith('ERROR:')) { throw response.replace('ERROR:', '').trim() } - schedule = response } catch (err) { if (!abortController.signal.aborted) { @@ -69,7 +75,7 @@ on:click={genLoading ? () => abortController?.abort() : undefined} /> - {#if $copilotInfo.exists_openai_resource_path} + {#if $copilotInfo.exists_ai_resource}

Enable Windmill AI in the workspace settings -{#if $copilotInfo.exists_openai_resource_path} +{#if $copilotInfo.exists_ai_resource} {@const fixAction = (_) => { - if ($copilotInfo.exists_openai_resource_path) { + if ($copilotInfo.exists_ai_resource) { onFix(() => close(null)) } }}

- {#if $copilotInfo.exists_openai_resource_path} + {#if $copilotInfo.exists_ai_resource}
{#if $generatedCode.length > 0}
@@ -211,7 +213,7 @@

Enable Windmill AI in the workspace settings

{/if}
- {:else if $copilotInfo.exists_openai_resource_path} + {:else if $copilotInfo.exists_ai_resource}
@@ -397,7 +406,14 @@
- GPT-4o + {#if $copilotInfo.ai_provider === 'openai'} + GPT-4o + {:else if $copilotInfo.ai_provider === 'anthropic'} + Claude-3.5 + {:else} + Codestral + {/if} +
@@ -493,7 +509,7 @@ {:else}

Enable Windmill AI in the diff --git a/frontend/src/lib/components/copilot/StepGen.svelte b/frontend/src/lib/components/copilot/StepGen.svelte index f2a7617b70..138a3b0109 100644 --- a/frontend/src/lib/components/copilot/StepGen.svelte +++ b/frontend/src/lib/components/copilot/StepGen.svelte @@ -70,7 +70,7 @@ } async function onGenerate() { - if (!selectedCompletion && !$copilotInfo.exists_openai_resource_path) { + if (!selectedCompletion && !$copilotInfo.exists_ai_resource) { sendUserToast( 'Windmill AI is not enabled, you can activate it in the workspace settings', true diff --git a/frontend/src/lib/components/copilot/StepInputGen.svelte b/frontend/src/lib/components/copilot/StepInputGen.svelte index f7b4d0010b..244c4386a2 100644 --- a/frontend/src/lib/components/copilot/StepInputGen.svelte +++ b/frontend/src/lib/components/copilot/StepInputGen.svelte @@ -1,7 +1,7 @@ -{#if $copilotInfo.exists_openai_resource_path && $stepInputCompletionEnabled} +{#if $copilotInfo.exists_ai_resource && $stepInputCompletionEnabled} { createFlowInput() diff --git a/frontend/src/lib/components/copilot/StepInputsGen.svelte b/frontend/src/lib/components/copilot/StepInputsGen.svelte index d1ed185515..7e14ffd659 100644 --- a/frontend/src/lib/components/copilot/StepInputsGen.svelte +++ b/frontend/src/lib/components/copilot/StepInputsGen.svelte @@ -7,7 +7,7 @@ import type { FlowEditorContext } from '../flows/types' import type { PickableProperties } from '../flows/previousResults' import { getContext } from 'svelte' - import { getNonStreamingCompletion } from './lib' + import { getNonStreamingCompletion, type AiProviderTypes } from './lib' import { sendUserToast } from '$lib/toast' import Button from '../common/button/Button.svelte' import type { FlowCopilotContext } from './flow' @@ -83,7 +83,7 @@ Your answer has to be in the following format (one line per input): input_name1: expression1 input_name2: expression2 ...` - + const aiProvider = $copilotInfo.ai_provider generatedContent = await getNonStreamingCompletion( [ { @@ -91,7 +91,8 @@ input_name2: expression2 content: user } ], - abortController + abortController, + aiProvider as AiProviderTypes ) parsedInputs = generatedContent.split('\n').map((x) => x.split(': ')) @@ -168,7 +169,7 @@ input_name2: expression2

- {#if $copilotInfo.exists_openai_resource_path && $stepInputCompletionEnabled} + {#if $copilotInfo.exists_ai_resource && $stepInputCompletionEnabled} { createFlowInputs() @@ -236,10 +237,10 @@ input_name2: expression2

- {#if !$copilotInfo.exists_openai_resource_path} + {#if !$copilotInfo.exists_ai_resource} Enable Windmill AI in the{' '} diff --git a/frontend/src/lib/components/copilot/TestOpenaiKey.svelte b/frontend/src/lib/components/copilot/TestAiKey.svelte similarity index 87% rename from frontend/src/lib/components/copilot/TestOpenaiKey.svelte rename to frontend/src/lib/components/copilot/TestAiKey.svelte index 9499a30cf7..23f3093eb3 100644 --- a/frontend/src/lib/components/copilot/TestOpenaiKey.svelte +++ b/frontend/src/lib/components/copilot/TestAiKey.svelte @@ -1,9 +1,10 @@ @@ -28,7 +29,8 @@ content: "this is a test, simply reply with 'ok'" } ], - abortController + abortController, + aiProvider }) sendUserToast('Valid key') } catch (err) { diff --git a/frontend/src/lib/components/copilot/completion.ts b/frontend/src/lib/components/copilot/completion.ts index ee94db6c1c..b30a3fe7e0 100644 --- a/frontend/src/lib/components/copilot/completion.ts +++ b/frontend/src/lib/components/copilot/completion.ts @@ -1,5 +1,5 @@ import type { ChatCompletionMessageParam } from 'openai/resources/chat/index.mjs' -import { getNonStreamingCompletion } from './lib' +import { getNonStreamingCompletion, type AiProviderTypes } from './lib' import { codeCompletionLoading } from '$lib/stores' const systemPrompt = `You are a code completion assistant, return the code that should go instead of the . @@ -73,7 +73,8 @@ export async function editorCodeCompletion( before: string, after: string, lang: string, - abortController: AbortController + abortController: AbortController, + aiProvider: AiProviderTypes ) { codeCompletionLoading.set(true) const messages: ChatCompletionMessageParam[] = [ @@ -91,7 +92,7 @@ export async function editorCodeCompletion( ] try { - const result = await getNonStreamingCompletion(messages, abortController) + const result = await getNonStreamingCompletion(messages, abortController, aiProvider) const match = result.match(/```[a-zA-Z]+\n([\s\S]*?)\n```/) diff --git a/frontend/src/lib/components/copilot/flow.ts b/frontend/src/lib/components/copilot/flow.ts index 502f1b0dc6..7867a2f2d5 100644 --- a/frontend/src/lib/components/copilot/flow.ts +++ b/frontend/src/lib/components/copilot/flow.ts @@ -6,7 +6,12 @@ import { type InputTransform, type Script } from '$lib/gen' -import { addResourceTypes, deltaCodeCompletion, getNonStreamingCompletion } from './lib' +import { + addResourceTypes, + deltaCodeCompletion, + getNonStreamingCompletion, + type AiProviderTypes +} from './lib' import type { Writable } from 'svelte/store' import type Editor from '../Editor.svelte' import type { Drawer } from '../common' @@ -14,7 +19,7 @@ import { scriptLangToEditorLang } from '$lib/scripts' export type FlowCopilotModule = { id: string - type: 'trigger' | 'script' + type: 'trigger' | 'script' description: string code: string source: 'hub' | 'custom' | undefined @@ -106,7 +111,6 @@ To maintain state across runs, you can use get_state() and set_state(value) whic {additionalInformation}` } - // const preprocessorPrompts: { // bun: string // python3: string @@ -128,7 +132,7 @@ To maintain state across runs, you can use get_state() and set_state(value) whic // headers: Record // } // }, -// /* your other args */ +// /* your other args */ // ) { // return { // // return the args to be passed to the flow @@ -168,7 +172,6 @@ To maintain state across runs, you can use get_state() and set_state(value) whic // {additionalInformation}` // } - const firstActionPrompt = `I'm building a workflow which is a sequence of script steps. Write a script in {codeLang} which should {description}. Return the script's output. @@ -255,7 +258,8 @@ export async function stepCopilot( }) | undefined, isFirstInLoop: boolean, - abortController: AbortController + abortController: AbortController, + aiProvider: AiProviderTypes ) { if (module.source !== 'custom') { throw new Error('Not a custom module') @@ -265,9 +269,9 @@ export async function stepCopilot( let prompt = module.type === 'trigger' ? triggerPrompts[lang] - // : module.type === 'preprocessor' + : // : module.type === 'preprocessor' // ? preprocessorPrompts[lang] - : pastModule === undefined + pastModule === undefined ? firstActionPrompt : isFirstInLoop ? loopActionPrompt @@ -310,7 +314,8 @@ export async function stepCopilot( } ], deltaCodeStore, - abortController + abortController, + aiProvider ) return code } @@ -322,7 +327,8 @@ export async function glueCopilot( value: RawScript | PathScript }, isFirstInLoop: boolean, - abortController: AbortController + abortController: AbortController, + aiProvider: AiProviderTypes ) { const { prevCode, prevLang } = await getPreviousStepContent(pastModule, workspace) @@ -357,7 +363,8 @@ export async function glueCopilot( ) } ], - abortController + abortController, + aiProvider ) const matches = response.matchAll(/([a-zA-Z_0-9.]+): (.+)/g) diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 332abd648f..6f44a8db20 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -1,42 +1,147 @@ import { OpenAI } from 'openai' import { OpenAPI, ResourceService, type Script } from '../../gen' import type { Writable } from 'svelte/store' - +import { Anthropic } from '@anthropic-ai/sdk' import type { DBSchema, GraphqlSchema, SQLSchema } from '$lib/stores' import { formatResourceTypes } from './utils' - import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' - +import { Mistral } from '@mistralai/mistralai' import { buildClientSchema, printSchema } from 'graphql' import type { ChatCompletionCreateParamsStreaming, ChatCompletionMessageParam } from 'openai/resources/index.mjs' +import type { MessageCreateParams, MessageParam } from '@anthropic-ai/sdk/resources/messages.mjs' +import type { ChatCompletionRequest } from '@mistralai/mistralai/models/components/chatcompletionrequest' +import type { + SystemMessage, + UserMessage, + AssistantMessage, + ToolMessage, + CompletionEvent, + ContentChunk +} from '@mistralai/mistralai/models/components' + export const SUPPORTED_LANGUAGES = new Set(Object.keys(GEN_CONFIG.prompts)) -const openaiConfig: ChatCompletionCreateParamsStreaming = { - temperature: 0, - max_tokens: 16384, - model: 'gpt-4o-2024-08-06', - seed: 42, - stream: true, - messages: [] +export type AiProviderTypes = 'openai' | 'anthropic' | 'mistral' + +interface AiProvider { + init: (workspace: string, updateClient: boolean, token?: string) => void } -class WorkspacedOpenai { +class WorkspacedMistral implements AiProvider { + private client: Mistral | undefined + + init(workspace: string, updateClient: boolean, token?: string) { + if (!this.client || updateClient) { + this.client = initWorkspaceAiProvider(workspace, 'mistral', token) as unknown as Mistral + } + } + + getClient() { + if (!this.client) { + throw new Error('AnthropicAi not initialized') + } + return this.client + } +} + +export namespace MistralAi { + export let workspace = new WorkspacedMistral() + + export const mistralConfig: ChatCompletionRequest = { + temperature: 0, + model: null, + maxTokens: 32000, + messages: [] + } + + export type MistralParamsMessage = + | (SystemMessage & { role: 'system' }) + | (UserMessage & { role: 'user' }) + | (AssistantMessage & { role: 'assistant' }) + | (ToolMessage & { role: 'tool' }) + + export function retrieveTextValue(chunks: string | ContentChunk[] | null | undefined): string { + let response = '' + if (Array.isArray(chunks)) { + for (const chunk of chunks) { + if (chunk.type === 'text') { + response += chunk.text + } + } + return response + } + return chunks as string + } +} + +class WorkspacedAnthropic implements AiProvider { + private client: Anthropic | undefined + + init(workspace: string, updateClient: boolean, token: string | undefined = undefined) { + if (!this.client || updateClient) { + this.client = initWorkspaceAiProvider(workspace, 'anthropic', token) as unknown as Anthropic + } + } + + getClient() { + if (!this.client) { + throw new Error('AnthropicAi not initialized') + } + return this.client + } +} + +export namespace AnthropicAi { + export let workspace = new WorkspacedAnthropic() + + export const config: MessageCreateParams = { + temperature: 0, + max_tokens: 8192, + model: 'claude-3-5-sonnet-20241022', + messages: [] + } + + export function getSystemPromptAndArrayMessages( + messages: ChatCompletionMessageParam[] + ): [string, MessageParam[]] { + let system: string | undefined = undefined + if (messages[0].role == 'system') { + system = messages[0].content as string + messages.shift() + } + const anthropicMessages: MessageParam[] = messages.map((message) => { + return { + role: message.role == 'user' ? 'user' : 'assistant', + content: message.content as string + } + }) + return [system as string, anthropicMessages ?? []] + } + + export function retrieveTextValue(part: Anthropic.Messages.RawMessageStreamEvent) { + let response = '' + if (part.type == 'content_block_delta') { + if (part.delta.type == 'text_delta') { + response = part.delta.text + } else { + response = part.delta.partial_json + } + } + return response + } +} + +class WorkspacedOpenai implements AiProvider { private client: OpenAI | undefined - init(workspace: string, token: string | undefined = undefined) { - const baseURL = `${location.origin}${OpenAPI.BASE}/w/${workspace}/openai/proxy` - this.client = new OpenAI({ - baseURL, - apiKey: 'fakekey', - defaultHeaders: { - Authorization: token ? `Bearer ${token}` : '' - }, - dangerouslyAllowBrowser: true - }) + init(workspace: string, updateClient: boolean, token: string | undefined = undefined) { + if (!this.client || updateClient) { + this.client = initWorkspaceAiProvider(workspace, 'openai', token) as unknown as OpenAI + } } getClient() { @@ -47,34 +152,141 @@ class WorkspacedOpenai { } } -export let workspacedOpenai = new WorkspacedOpenai() +export namespace OpenAi { + export let workspace = new WorkspacedOpenai() + + export const openaiConfig: ChatCompletionCreateParamsStreaming = { + temperature: 0, + max_tokens: 16384, + model: 'gpt-4o-2024-08-06', + seed: 42, + stream: true, + messages: [] + } + + export function retrieveTextValue(part: OpenAI.Chat.Completions.ChatCompletionChunk) { + return part.choices[0]?.delta?.content || '' + } +} + +export function initAllAiWorkspace(workspace: string, updateClient: boolean = false) { + OpenAi.workspace.init(workspace, updateClient) + AnthropicAi.workspace.init(workspace, updateClient) + MistralAi.workspace.init(workspace, updateClient) +} + +function initWorkspaceAiProvider( + workspace: string, + aiProvider: AiProviderTypes, + token: string | undefined = undefined +): Anthropic | OpenAI | Mistral { + const baseURL = `${location.origin}${OpenAPI.BASE}/w/${workspace}/ai/proxy` + let client + switch (aiProvider) { + case 'openai': { + client = new OpenAI({ + baseURL, + apiKey: 'fake-key', + defaultHeaders: { + Authorization: token ? `Bearer ${token}` : '' + }, + dangerouslyAllowBrowser: true + }) + break + } + case 'anthropic': { + client = new Anthropic({ + baseURL, + apiKey: 'fake-key', + defaultHeaders: { + Authorization: token ? `Bearer ${token}` : '' + }, + dangerouslyAllowBrowser: true + }) + break + } + case 'mistral': { + client = new Mistral({ + serverURL: baseURL + }) + } + } + return client +} export async function testKey({ apiKey, abortController, - messages + messages, + aiProvider }: { apiKey?: string messages: ChatCompletionMessageParam[] abortController: AbortController + aiProvider: AiProviderTypes }) { if (apiKey) { - const openai = new OpenAI({ - apiKey, - dangerouslyAllowBrowser: true - }) - await openai.chat.completions.create( - { - ...openaiConfig, - messages, - stream: false - }, - { - signal: abortController.signal + switch (aiProvider) { + case 'openai': { + const openai = new OpenAI({ + apiKey, + dangerouslyAllowBrowser: true + }) + await openai.chat.completions.create( + { + ...OpenAi.openaiConfig, + messages, + stream: false + }, + { + signal: abortController.signal + } + ) + break } - ) + case 'anthropic': { + const anthropic = new Anthropic({ + apiKey, + dangerouslyAllowBrowser: true + }) + const [, anthropicMessages] = AnthropicAi.getSystemPromptAndArrayMessages(messages) + await anthropic.messages.create( + { + ...AnthropicAi.config, + messages: anthropicMessages, + stream: false + }, + { + signal: abortController.signal + } + ) + break + } + case 'mistral': { + const mistral = new Mistral({ + apiKey + }) + await mistral.chat.complete( + { + ...MistralAi.mistralConfig, + model: 'codestral-latest', + stream: false, + messages: messages as MistralAi.MistralParamsMessage[] + }, + { + fetchOptions: { + signal: abortController.signal, + headers: { + 'content-type': 'application/json' + } + } + } + ) + break + } + } } else { - await getNonStreamingCompletion(messages, abortController, undefined, true) + await getNonStreamingCompletion(messages, abortController, aiProvider, undefined, true) } } @@ -255,57 +467,154 @@ const PROMPTS_CONFIGS = { export async function getNonStreamingCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, - model = openaiConfig.model, + aiProvider: AiProviderTypes, + model = OpenAi.openaiConfig.model, noCache?: boolean ) { - const openaiClient = workspacedOpenai.getClient() - const completion = await openaiClient.chat.completions.create( - { - ...openaiConfig, - messages, - stream: false, - model + let response: string | undefined = '' + const queryOptions = { + query: { + no_cache: noCache }, - { - query: { - no_cache: noCache - }, - signal: abortController.signal + signal: abortController.signal + } + switch (aiProvider) { + case 'openai': { + const openaiClient = OpenAi.workspace.getClient() + const completion = await openaiClient.chat.completions.create( + { + ...OpenAi.openaiConfig, + messages, + stream: false, + model + }, + queryOptions + ) + response = completion.choices[0]?.message.content || '' + break } - ) - - // if (completion.usage) { - // const { prompt_tokens, completion_tokens } = completion.usage - // console.log('Cost: ', (prompt_tokens * 0.0015 + completion_tokens * 0.002) / 1000) - // } - - return completion.choices[0]?.message.content || '' + case 'anthropic': { + const anthropicClient = AnthropicAi.workspace.getClient() + const [system, anthropicMessages] = AnthropicAi.getSystemPromptAndArrayMessages(messages) + const message = await anthropicClient.messages.create( + { + ...AnthropicAi.config, + system, + messages: anthropicMessages, + stream: false + }, + queryOptions + ) + response = message.content[0].type === 'text' ? message.content[0].text : '' + break + } + case 'mistral': { + const mistralClient = MistralAi.workspace.getClient() + const message = await mistralClient.chat.complete( + { + ...MistralAi.mistralConfig, + model: 'codestral-latest', + stream: false, + messages: messages as MistralAi.MistralParamsMessage[] + }, + { + fetchOptions: { + signal: abortController.signal, + cache: 'no-store' + } + } + ) + response = MistralAi.retrieveTextValue(message.choices && message.choices[0].message.content) + break + } + } + return response } export async function getCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, - model = openaiConfig.model + aiProvider: AiProviderTypes, + model = OpenAi.openaiConfig.model ) { - const openaiClient = workspacedOpenai.getClient() - const completion = await openaiClient.chat.completions.create( - { - ...openaiConfig, - messages, - model - }, - { - signal: abortController.signal - } - ) + switch (aiProvider) { + case 'anthropic': { + const anthropicClient = AnthropicAi.workspace.getClient() + const [system, anthropicMessages] = AnthropicAi.getSystemPromptAndArrayMessages(messages) - return completion + const completion = await anthropicClient.messages.create( + { + ...AnthropicAi.config, + system, + messages: anthropicMessages, + stream: true + }, + { signal: abortController.signal } + ) + return completion + } + case 'openai': { + const openaiClient = OpenAi.workspace.getClient() + const completion = await openaiClient.chat.completions.create( + { + ...OpenAi.openaiConfig, + messages, + model + }, + { + signal: abortController.signal + } + ) + return completion + } + case 'mistral': { + const mistralClient = MistralAi.workspace.getClient() + const message = await mistralClient.chat.stream( + { + ...MistralAi.mistralConfig, + model: 'codestral-latest', + messages: messages as MistralAi.MistralParamsMessage[] + }, + { + fetchOptions: { + signal: abortController.signal, + cache: 'no-store' + } + } + ) + return message + } + } +} + +export function getResponseFromEvent( + part: + | Anthropic.Messages.RawMessageStreamEvent + | OpenAI.Chat.Completions.ChatCompletionChunk + | CompletionEvent, + aiProvider: AiProviderTypes +): string { + switch (aiProvider) { + case 'openai': { + const messages = part as OpenAI.Chat.Completions.ChatCompletionChunk + return OpenAi.retrieveTextValue(messages) + } + case 'anthropic': { + const messages = part as Anthropic.Messages.RawMessageStreamEvent + return AnthropicAi.retrieveTextValue(messages) + } + case 'mistral': { + const messages = part as CompletionEvent + return MistralAi.retrieveTextValue(messages.data.choices[0].delta.content) + } + } } export async function copilot( scriptOptions: CopilotOptions, generatedCode: Writable, abortController: AbortController, + aiProvider: AiProviderTypes, generatedExplanation?: Writable ) { const { prompt, systemPrompt } = await getPrompts(scriptOptions) @@ -321,13 +630,14 @@ export async function copilot( content: prompt } ], - abortController + abortController, + aiProvider ) let response = '' let code = '' for await (const part of completion) { - response += part.choices[0]?.delta?.content || '' + response += getResponseFromEvent(part, aiProvider) let match = response.match(/```[a-zA-Z]+\n([\s\S]*?)\n```/) if (match) { @@ -393,15 +703,16 @@ function getStringEndDelta(prev: string, now: string) { export async function deltaCodeCompletion( messages: ChatCompletionMessageParam[], generatedCodeDelta: Writable, - abortController: AbortController + abortController: AbortController, + aiProvider: AiProviderTypes ) { - const completion = await getCompletion(messages, abortController) + const completion = await getCompletion(messages, abortController, aiProvider) let response = '' let code = '' let delta = '' for await (const part of completion) { - response += part.choices[0]?.delta?.content || '' + response += getResponseFromEvent(part, aiProvider) let match = response.match(/```[a-zA-Z]+\n([\s\S]*?)\n```/) if (match) { diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index 4439009115..a85814a5e1 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -87,7 +87,7 @@ } async function onGenerate() { - if (!selectedCompletion && !$copilotInfo.exists_openai_resource_path) { + if (!selectedCompletion && !$copilotInfo.exists_ai_resource) { sendUserToast( 'Windmill AI is not enabled, you can activate it in the workspace settings', true diff --git a/frontend/src/lib/components/flows/map/FlowCopilotButton.svelte b/frontend/src/lib/components/flows/map/FlowCopilotButton.svelte index 53f78cce59..51cbec517e 100644 --- a/frontend/src/lib/components/flows/map/FlowCopilotButton.svelte +++ b/frontend/src/lib/components/flows/map/FlowCopilotButton.svelte @@ -20,7 +20,7 @@

- {#if !$copilotInfo.exists_openai_resource_path} + {#if !$copilotInfo.exists_ai_resource}

Enable Windmill AI in the { diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte index 220b597384..8e53a1030c 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte @@ -91,10 +91,7 @@ kind ) } else if (kind == 'forloop') { - ;[module, state] = await createLoop( - module.id, - !disableAi && $copilotInfo.exists_openai_resource_path - ) + ;[module, state] = await createLoop(module.id, !disableAi && $copilotInfo.exists_ai_resource) } else if (kind == 'whileloop') { ;[module, state] = await createWhileLoop(module.id) } else if (kind == 'branchone') { diff --git a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte index b4645516a0..b5b44504d0 100644 --- a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte +++ b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte @@ -19,6 +19,7 @@ import MenuButton from './MenuButton.svelte' import { MenuItem } from '@rgossiaux/svelte-headlessui' import { isCloudHosted } from '$lib/cloud' + import { initAllAiWorkspace } from '../copilot/lib' export let isCollapsed: boolean = false @@ -26,7 +27,7 @@ if ($workspaceStore === id) { return } - + initAllAiWorkspace(id, true) const editPages = [ '/scripts/edit/', '/flows/edit/', diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 9d8af53bae..38370febe9 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -70,10 +70,12 @@ export const userWorkspaces: Readable< } }) export const copilotInfo = writable<{ - exists_openai_resource_path: boolean + ai_provider: string + exists_ai_resource: boolean code_completion_enabled: boolean }>({ - exists_openai_resource_path: false, + ai_provider: '', + exists_ai_resource: false, code_completion_enabled: false }) export const codeCompletionLoading = writable(false) diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 99fc3c0a0c..1884757ab4 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -43,7 +43,7 @@ import { syncTutorialsTodos } from '$lib/tutorialUtils' import { ArrowLeft, Search } from 'lucide-svelte' import { getUserExt } from '$lib/user' - import { workspacedOpenai } from '$lib/components/copilot/lib' + import { initAllAiWorkspace } from '$lib/components/copilot/lib' import { twMerge } from 'tailwind-merge' import OperatorMenu from '$lib/components/sidebar/OperatorMenu.svelte' import GlobalSearchModal from '$lib/components/search/GlobalSearchModal.svelte' @@ -230,12 +230,14 @@ let devOnly = $page.url.pathname.startsWith(base + '/scripts/dev') async function loadCopilot(workspace: string) { - workspacedOpenai.init(workspace) + initAllAiWorkspace(workspace) try { copilotInfo.set(await WorkspaceService.getCopilotInfo({ workspace })) } catch (err) { + console.log(err) copilotInfo.set({ - exists_openai_resource_path: false, + ai_provider: '', + exists_ai_resource: false, code_completion_enabled: false }) console.error('Could not get copilot info') diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/create_workspace/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/create_workspace/+page.svelte index 71eefb246d..fd47e0f011 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/create_workspace/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/create_workspace/+page.svelte @@ -18,11 +18,12 @@ import Tooltip from '$lib/components/Tooltip.svelte' import { onMount } from 'svelte' import { sendUserToast } from '$lib/toast' - import TestOpenaiKey from '$lib/components/copilot/TestOpenaiKey.svelte' + import TestAiKey from '$lib/components/copilot/TestAiKey.svelte' import { switchWorkspace } from '$lib/storeUtils' import { isCloudHosted } from '$lib/cloud' import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' + import type { AiProviderTypes } from '$lib/components/copilot/lib' const rd = $page.url.searchParams.get('rd') @@ -32,7 +33,7 @@ let errorId = '' let errorUser = '' - let openAiKey = '' + let aiKey = '' let codeCompletionEnabled = true let checking = false @@ -68,7 +69,7 @@ requestBody: { operator: operatorOnly, invite_all: !isCloudHosted(), auto_add: true } }) } - if (openAiKey != '') { + if (aiKey != '') { let actualUsername = username if (automateUsernameCreation) { const user = await UserService.whoami({ @@ -76,14 +77,14 @@ }) actualUsername = user.username } - let path = `u/${actualUsername}/openai_windmill_codegen` + let path = `u/${actualUsername}/${selected}_windmill_codegen` await VariableService.createVariable({ workspace: id, requestBody: { path, - value: openAiKey, + value: aiKey, is_secret: true, - description: 'Token for openai' + description: 'Ai token' } }) await ResourceService.createResource({ @@ -93,12 +94,15 @@ value: { api_key: '$var:' + path }, - resource_type: 'openai' + resource_type: selected } }) await WorkspaceService.editCopilotConfig({ workspace: id, - requestBody: { openai_resource_path: path, code_completion_enabled: codeCompletionEnabled } + requestBody: { + ai_resource: { path, provider: selected }, + code_completion_enabled: codeCompletionEnabled + } }) } @@ -164,6 +168,7 @@ let auto_invite = false let operatorOnly = false + let selected: AiProviderTypes = 'openai' @@ -191,30 +196,38 @@ {/if} in the docs - - (optional but recommended) - -

+
+ + AI key for Windmill AI + + Find out how it can help you in the docs + + (optional but recommended) + +
+ + + + + +
+
+
- +
- {#if openAiKey} + {#if aiKey} diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 14be8dbcaa..71d868779e 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -43,7 +43,7 @@ import PremiumInfo from '$lib/components/settings/PremiumInfo.svelte' import Toggle from '$lib/components/Toggle.svelte' - import TestOpenaiKey from '$lib/components/copilot/TestOpenaiKey.svelte' + import TestAiKey from '$lib/components/copilot/TestAiKey.svelte' import Portal from '$lib/components/Portal.svelte' import { fade } from 'svelte/transition' @@ -56,6 +56,9 @@ } from '$lib/workspace_settings' import { base } from '$lib/base' import { hubPaths } from '$lib/hub' + import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' + import type { AiProviderTypes } from '$lib/components/copilot/lib' type GitSyncTypeMap = { scripts: boolean @@ -99,7 +102,8 @@ let errorHandlerItemKind: 'flow' | 'script' = 'script' let errorHandlerExtraArgs: Record = {} let errorHandlerMutedOnCancel: boolean | undefined = undefined - let openaiResourceInitialPath: string | undefined = undefined + let aiResourceInitialPath: string | undefined = undefined + let aiResourceInitialProvider: string | undefined = undefined let s3ResourceSettings: S3ResourceSettings = { resourceType: 's3', @@ -128,6 +132,7 @@ let workspaceReencryptionInProgress: boolean = false let encryptionKeyRegex = /^[a-zA-Z0-9]{64}$/ let codeCompletionEnabled: boolean = false + let selected: AiProviderTypes = 'openai' let tab = ($page.url.searchParams.get('tab') as | 'users' @@ -175,31 +180,37 @@ } } - async function editCopilotConfig(openaiResourcePath: string): Promise { + async function editCopilotConfig(aiResourcePath: string, aiProvider: string): Promise { // in JS, an empty string is also falsy - openaiResourceInitialPath = openaiResourcePath - if (openaiResourcePath) { + aiResourceInitialPath = aiResourcePath + aiResourceInitialProvider = aiProvider + if (aiResourcePath) { await WorkspaceService.editCopilotConfig({ workspace: $workspaceStore!, requestBody: { - openai_resource_path: openaiResourcePath, + ai_resource: { + path: aiResourcePath, + provider: aiProvider + }, code_completion_enabled: codeCompletionEnabled } }) copilotInfo.set({ - exists_openai_resource_path: true, + ai_provider: aiProvider, + exists_ai_resource: true, code_completion_enabled: codeCompletionEnabled }) } else { await WorkspaceService.editCopilotConfig({ workspace: $workspaceStore!, requestBody: { - openai_resource_path: undefined, + ai_resource: undefined, code_completion_enabled: codeCompletionEnabled } }) copilotInfo.set({ - exists_openai_resource_path: true, + ai_provider: '', + exists_ai_resource: false, code_completion_enabled: codeCompletionEnabled }) } @@ -373,7 +384,9 @@ customer_id = settings.customer_id workspaceToDeployTo = settings.deploy_to webhook = settings.webhook - openaiResourceInitialPath = settings.openai_resource_path + aiResourceInitialPath = settings.ai_resource?.path + aiResourceInitialProvider = settings.ai_resource?.provider + selected = aiResourceInitialProvider as AiProviderTypes ?? 'openai' errorHandlerItemKind = settings.error_handler?.split('/')[0] as 'flow' | 'script' errorHandlerScriptPath = (settings.error_handler ?? '').split('/').slice(1).join('/') errorHandlerInitialScriptPath = errorHandlerScriptPath @@ -631,7 +644,7 @@
Error Handler
 - +
Windmill AI
 @@ -959,7 +972,7 @@ Save
- {:else if tab == 'openai'} + {:else if tab == 'ai'}
Windmill AI
@@ -967,7 +980,7 @@ Select an OpenAI resource to unlock Windmill AI features.
- Windmill AI uses OpenAI's GPT-4o for all AI features. + Windmill AI supports integration with your preferred AI provider for all AI features.
+ { + aiResourceInitialPath = '' + }} + > + + + +
- {#key [openaiResourceInitialPath, usingOpenaiClientCredentialsOauth]} + {#key [aiResourceInitialPath, aiResourceInitialProvider, usingOpenaiClientCredentialsOauth, selected]} { - editCopilotConfig(ev.detail) + editCopilotConfig(ev.detail, selected) }} /> + {/key} -
{ - editCopilotConfig(openaiResourceInitialPath || '') + editCopilotConfig(aiResourceInitialPath || '', aiResourceInitialProvider || '') }} />
From 180bb826436f9960def8c6cf3e9692714ffdf917 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 14 Nov 2024 16:09:34 +0100 Subject: [PATCH 07/31] fix: autoscaling when count < min worker set to min_workers --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 498ca96e9b..e784771cb7 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0d9c8813acd28848515c736e7b684220b5a785a3 \ No newline at end of file +15602d5972e79192ff8f121cd783613b8ab365fa \ No newline at end of file From 15f1b7cf7d1af1fb740548207b5e1f38ff83ff37 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 14 Nov 2024 16:21:07 +0100 Subject: [PATCH 08/31] update openapi --- backend/windmill-api/openapi-deref.json | 238 ++- backend/windmill-api/openapi-deref.yaml | 2090 ++++++++++++----------- 2 files changed, 1344 insertions(+), 984 deletions(-) diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index a6e225259a..91214e99b4 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -1,7 +1,7 @@ { "openapi": "3.0.3", "info": { - "version": "1.418.0", + "version": "1.423.2", "title": "Windmill API", "contact": { "name": "Windmill Team", @@ -286,7 +286,7 @@ } } }, - "/w/{workspace}/users/{username}": { + "/w/{workspace}/users/get/{username}": { "get": { "summary": "get user (require admin privilege)", "operationId": "getUser", @@ -436,6 +436,106 @@ } } }, + "/users/set_password_of/{user}": { + "post": { + "summary": "set password for a specific user (require super admin)", + "operationId": "setPasswordForUser", + "tags": [ + "user" + ], + "parameters": [ + { + "name": "user", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "set password", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "password": { + "type": "string" + } + }, + "required": [ + "password" + ] + } + } + } + }, + "responses": { + "200": { + "description": "password set", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/users/set_login_type/{user}": { + "post": { + "summary": "set login type for a specific user (require super admin)", + "operationId": "setLoginTypeForUser", + "tags": [ + "user" + ], + "parameters": [ + { + "name": "user", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "set login type", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "login_type": { + "type": "string" + } + }, + "required": [ + "login_type" + ] + } + } + } + }, + "responses": { + "200": { + "description": "login type set", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/users/create": { "post": { "summary": "create user", @@ -1105,6 +1205,9 @@ }, "tls_implicit": { "type": "boolean" + }, + "disable_tls": { + "type": "boolean" } }, "required": [ @@ -1113,7 +1216,8 @@ "password", "port", "from", - "tls_implicit" + "tls_implicit", + "disable_tls" ] } }, @@ -2460,8 +2564,8 @@ "deploy_to": { "type": "string" }, - "openai_resource_path": { - "type": "string" + "ai_resource": { + "$ref": "#/components/schemas/AiResource" }, "code_completion_enabled": { "type": "boolean" @@ -2901,8 +3005,8 @@ "code_completion_enabled" ], "properties": { - "openai_resource_path": { - "type": "string" + "ai_resource": { + "$ref": "#/components/schemas/AiResource" }, "code_completion_enabled": { "type": "boolean" @@ -2946,7 +3050,10 @@ "schema": { "type": "object", "properties": { - "exists_openai_resource_path": { + "ai_provider": { + "type": "string" + }, + "exists_ai_resource": { "type": "boolean" }, "code_completion_enabled": { @@ -2954,7 +3061,8 @@ } }, "required": [ - "exists_openai_resource_path", + "ai_provider", + "exists_ai_resource", "code_completion_enabled" ] } @@ -13160,6 +13268,30 @@ } } }, + "/workers/queue_counts": { + "get": { + "summary": "get counts of jobs waiting for an executor per tag", + "operationId": "getCountsOfJobsWaitingPerTag", + "tags": [ + "worker" + ], + "responses": { + "200": { + "description": "queue counts", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + } + } + } + } + } + } + }, "/configs/list_worker_groups": { "get": { "summary": "list worker groups", @@ -14810,6 +14942,20 @@ "schema": { "type": "string" } + }, + { + "name": "content_type", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "content_disposition", + "in": "query", + "schema": { + "type": "string" + } } ], "requestBody": { @@ -16116,6 +16262,21 @@ } }, "schemas": { + "AiResource": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "provider": { + "type": "string" + } + }, + "required": [ + "path", + "provider" + ] + }, "Script": { "type": "object", "properties": { @@ -17226,7 +17387,7 @@ "jobs.disapproval", "jobs.delete", "account.delete", - "openai.request", + "ai.request", "resources.create", "resources.update", "resources.delete", @@ -18067,6 +18228,23 @@ "route_path": { "type": "string" }, + "static_asset_config": { + "type": "object", + "properties": { + "s3": { + "type": "string" + }, + "storage": { + "type": "string" + }, + "filename": { + "type": "string" + } + }, + "required": [ + "s3" + ] + }, "is_flow": { "type": "boolean" }, @@ -18126,6 +18304,23 @@ "route_path": { "type": "string" }, + "static_asset_config": { + "type": "object", + "properties": { + "s3": { + "type": "string" + }, + "storage": { + "type": "string" + }, + "filename": { + "type": "string" + } + }, + "required": [ + "s3" + ] + }, "is_flow": { "type": "boolean" }, @@ -18168,6 +18363,23 @@ "route_path": { "type": "string" }, + "static_asset_config": { + "type": "object", + "properties": { + "s3": { + "type": "string" + }, + "storage": { + "type": "string" + }, + "filename": { + "type": "string" + } + }, + "required": [ + "s3" + ] + }, "is_flow": { "type": "boolean" }, @@ -18917,6 +19129,12 @@ "type": "object" } }, + "s3_inputs": { + "type": "array", + "items": { + "type": "object" + } + }, "execution_mode": { "type": "string", "enum": [ diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 94206795c2..0e7502d6cd 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - version: 1.418.0 + version: 1.423.2 title: Windmill API contact: name: Windmill Team @@ -87,7 +87,7 @@ paths: - name: id in: path required: true - schema: &ref_28 + schema: &ref_30 type: integer responses: '200': @@ -123,7 +123,7 @@ paths: - jobs.disapproval - jobs.delete - account.delete - - openai.request + - ai.request - resources.create - resources.update - resources.delete @@ -233,24 +233,24 @@ paths: - name: before description: filter on started before (inclusive) timestamp in: query - schema: &ref_118 + schema: &ref_120 type: string format: date-time - name: after description: filter on created after (exclusive) timestamp in: query - schema: &ref_119 + schema: &ref_121 type: string format: date-time - name: username description: filter on exact username of user in: query - schema: &ref_128 + schema: &ref_130 type: string - name: operation description: filter on exact or prefix name of operation in: query - schema: &ref_129 + schema: &ref_131 type: string - name: operations in: query @@ -265,12 +265,12 @@ paths: - name: resource description: filter on exact or prefix name of resource in: query - schema: &ref_130 + schema: &ref_132 type: string - name: action_kind description: filter on type of operation in: query - schema: &ref_131 + schema: &ref_133 type: string enum: - Create @@ -302,12 +302,12 @@ paths: application/json: schema: type: object - properties: &ref_143 + properties: &ref_145 email: type: string password: type: string - required: &ref_144 + required: &ref_146 - email - password responses: @@ -344,7 +344,7 @@ paths: text/plain: schema: type: string - /w/{workspace}/users/{username}: + /w/{workspace}/users/get/{username}: get: summary: get user (require admin privilege) operationId: getUser @@ -430,7 +430,7 @@ paths: application/json: schema: type: object - properties: &ref_145 + properties: &ref_147 is_admin: type: boolean operator: @@ -458,7 +458,7 @@ paths: - name: path in: path required: true - schema: &ref_21 + schema: &ref_23 type: string responses: '200': @@ -492,6 +492,68 @@ paths: text/plain: schema: type: string + /users/set_password_of/{user}: + post: + summary: set password for a specific user (require super admin) + operationId: setPasswordForUser + tags: + - user + parameters: + - name: user + in: path + required: true + schema: + type: string + requestBody: + description: set password + required: true + content: + application/json: + schema: + type: object + properties: + password: + type: string + required: + - password + responses: + '200': + description: password set + content: + text/plain: + schema: + type: string + /users/set_login_type/{user}: + post: + summary: set login type for a specific user (require super admin) + operationId: setLoginTypeForUser + tags: + - user + parameters: + - name: user + in: path + required: true + schema: + type: string + requestBody: + description: set login type + required: true + content: + application/json: + schema: + type: object + properties: + login_type: + type: string + required: + - login_type + responses: + '200': + description: login type set + content: + text/plain: + schema: + type: string /users/create: post: summary: create user @@ -784,7 +846,7 @@ paths: application/json: schema: type: object - properties: &ref_183 + properties: &ref_185 email: type: string workspaces: @@ -802,7 +864,7 @@ paths: - id - name - username - required: &ref_184 + required: &ref_186 - email - workspaces /workspaces/list_as_superadmin: @@ -844,14 +906,14 @@ paths: application/json: schema: type: object - properties: &ref_185 + properties: &ref_187 id: type: string name: type: string username: type: string - required: &ref_186 + required: &ref_188 - id - name responses: @@ -1000,6 +1062,8 @@ paths: type: string tls_implicit: type: boolean + disable_tls: + type: boolean required: - host - username @@ -1007,6 +1071,7 @@ paths: - port - from - tls_implicit + - disable_tls required: - to - smtp @@ -1081,7 +1146,7 @@ paths: type: array items: type: object - properties: &ref_228 + properties: &ref_230 id: type: integer description: Unique identifier for the alert @@ -1291,12 +1356,12 @@ paths: type: array items: type: object - properties: &ref_221 + properties: &ref_223 name: type: string value: type: object - required: &ref_222 + required: &ref_224 - name - value /users/email: @@ -1961,20 +2026,28 @@ paths: type: string deploy_to: type: string - openai_resource_path: - type: string + ai_resource: + type: object + properties: &ref_16 + path: + type: string + provider: + type: string + required: &ref_17 + - path + - provider code_completion_enabled: type: boolean error_handler: type: string error_handler_extra_args: type: object - additionalProperties: &ref_16 {} + additionalProperties: &ref_18 {} error_handler_muted_on_cancel: type: boolean large_file_storage: type: object - properties: &ref_17 + properties: &ref_19 type: type: string enum: @@ -2008,7 +2081,7 @@ paths: type: boolean git_sync: type: object - properties: &ref_18 + properties: &ref_20 include_path: type: array items: @@ -2033,7 +2106,7 @@ paths: type: array items: type: object - properties: &ref_205 + properties: &ref_207 script_path: type: string git_repo_resource_path: @@ -2058,12 +2131,12 @@ paths: - schedule - user - group - required: &ref_206 + required: &ref_208 - script_path - git_repo_resource_path deploy_ui: type: object - properties: &ref_19 + properties: &ref_21 include_path: type: array items: @@ -2083,7 +2156,7 @@ paths: type: string default_scripts: type: object - properties: &ref_20 + properties: &ref_22 order: type: array items: @@ -2371,8 +2444,10 @@ paths: required: - code_completion_enabled properties: - openai_resource_path: - type: string + ai_resource: + type: object + properties: *ref_16 + required: *ref_17 code_completion_enabled: type: boolean responses: @@ -2401,12 +2476,15 @@ paths: schema: type: object properties: - exists_openai_resource_path: + ai_provider: + type: string + exists_ai_resource: type: boolean code_completion_enabled: type: boolean required: - - exists_openai_resource_path + - ai_provider + - exists_ai_resource - code_completion_enabled /w/{workspace}/workspaces/edit_error_handler: post: @@ -2431,7 +2509,7 @@ paths: type: string error_handler_extra_args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 error_handler_muted_on_cancel: type: boolean responses: @@ -2462,7 +2540,7 @@ paths: properties: large_file_storage: type: object - properties: *ref_17 + properties: *ref_19 responses: '200': description: status @@ -2490,7 +2568,7 @@ paths: properties: git_sync_settings: type: object - properties: *ref_18 + properties: *ref_20 responses: '200': description: status @@ -2518,7 +2596,7 @@ paths: properties: deploy_ui_settings: type: object - properties: *ref_19 + properties: *ref_21 responses: '200': description: status @@ -2570,7 +2648,7 @@ paths: application/json: schema: type: object - properties: *ref_20 + properties: *ref_22 responses: '200': description: status @@ -2595,7 +2673,7 @@ paths: application/json: schema: type: object - properties: *ref_20 + properties: *ref_22 /w/{workspace}/workspaces/set_environment_variable: post: summary: set environment variable @@ -2721,7 +2799,7 @@ paths: application/json: schema: type: object - properties: *ref_17 + properties: *ref_19 /w/{workspace}/workspaces/usage: get: summary: get usage @@ -2808,7 +2886,7 @@ paths: type: array items: type: object - properties: &ref_142 + properties: &ref_144 email: type: string executions: @@ -2869,7 +2947,7 @@ paths: application/json: schema: type: object - properties: &ref_146 + properties: &ref_148 label: type: string expiration: @@ -2901,7 +2979,7 @@ paths: application/json: schema: type: object - properties: &ref_147 + properties: &ref_149 label: type: string expiration: @@ -2911,7 +2989,7 @@ paths: type: string workspace_id: type: string - required: &ref_148 + required: &ref_150 - impersonate_email responses: '201': @@ -2967,7 +3045,7 @@ paths: type: array items: type: object - properties: &ref_38 + properties: &ref_40 label: type: string expiration: @@ -2987,7 +3065,7 @@ paths: type: string email: type: string - required: &ref_39 + required: &ref_41 - token_prefix - created_at - last_used_at @@ -3036,7 +3114,7 @@ paths: application/json: schema: type: object - properties: &ref_151 + properties: &ref_153 path: type: string value: @@ -3052,7 +3130,7 @@ paths: expires_at: type: string format: date-time - required: &ref_152 + required: &ref_154 - path - value - is_secret @@ -3103,7 +3181,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: variable deleted @@ -3125,7 +3203,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 - name: already_encrypted in: query schema: @@ -3137,7 +3215,7 @@ paths: application/json: schema: type: object - properties: &ref_153 + properties: &ref_155 path: type: string value: @@ -3167,7 +3245,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 - name: decrypt_secret description: | ask to decrypt secret if this variable is secret @@ -3189,7 +3267,7 @@ paths: application/json: schema: type: object - properties: &ref_22 + properties: &ref_24 workspace_id: type: string path: @@ -3219,7 +3297,7 @@ paths: expires_at: type: string format: date-time - required: &ref_23 + required: &ref_25 - workspace_id - path - is_secret @@ -3238,7 +3316,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: variable @@ -3260,7 +3338,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: variable @@ -3300,8 +3378,8 @@ paths: type: array items: type: object - properties: *ref_22 - required: *ref_23 + properties: *ref_24 + required: *ref_25 /w/{workspace}/variables/list_contextual: get: summary: list contextual variables @@ -3322,7 +3400,7 @@ paths: type: array items: type: object - properties: &ref_149 + properties: &ref_151 name: type: string value: @@ -3331,7 +3409,7 @@ paths: type: string is_custom: type: boolean - required: &ref_150 + required: &ref_152 - name - value - description @@ -3347,7 +3425,7 @@ paths: - name: client_name in: path required: true - schema: &ref_24 + schema: &ref_26 type: string requestBody: description: Partially filled script @@ -3448,7 +3526,7 @@ paths: - name: client_name in: path required: true - schema: *ref_24 + schema: *ref_26 requestBody: description: code endpoint required: true @@ -3471,7 +3549,7 @@ paths: application/json: schema: type: object - properties: &ref_198 + properties: &ref_200 access_token: type: string expires_in: @@ -3482,7 +3560,7 @@ paths: type: array items: type: string - required: &ref_199 + required: &ref_201 - access_token /w/{workspace}/oauth/create_account: post: @@ -3533,7 +3611,7 @@ paths: - name: id in: path required: true - schema: &ref_25 + schema: &ref_27 type: integer requestBody: description: variable path @@ -3568,7 +3646,7 @@ paths: - name: id in: path required: true - schema: *ref_25 + schema: *ref_27 responses: '200': description: disconnected client @@ -3687,7 +3765,7 @@ paths: application/json: schema: type: object - properties: &ref_160 + properties: &ref_162 path: type: string value: {} @@ -3695,7 +3773,7 @@ paths: type: string resource_type: type: string - required: &ref_161 + required: &ref_163 - path - value - resource_type @@ -3720,7 +3798,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: resource deleted @@ -3742,7 +3820,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 requestBody: description: updated resource required: true @@ -3750,7 +3828,7 @@ paths: application/json: schema: type: object - properties: &ref_162 + properties: &ref_164 path: type: string description: @@ -3777,7 +3855,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 requestBody: description: updated resource required: true @@ -3808,7 +3886,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: resource @@ -3816,7 +3894,7 @@ paths: application/json: schema: type: object - properties: &ref_163 + properties: &ref_165 workspace_id: type: string path: @@ -3837,7 +3915,7 @@ paths: edited_at: type: string format: date-time - required: &ref_164 + required: &ref_166 - path - resource_type - is_oauth @@ -3855,7 +3933,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 - name: job_id description: job id in: query @@ -3882,7 +3960,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: resource value @@ -3903,7 +3981,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: does resource exists @@ -3953,7 +4031,7 @@ paths: type: array items: type: object - properties: &ref_165 + properties: &ref_167 workspace_id: type: string path: @@ -3984,7 +4062,7 @@ paths: edited_at: type: string format: date-time - required: &ref_166 + required: &ref_168 - path - resource_type - is_oauth @@ -4031,7 +4109,7 @@ paths: - name: name in: path required: true - schema: &ref_101 + schema: &ref_103 type: string responses: '200': @@ -4068,7 +4146,7 @@ paths: application/json: schema: type: object - properties: &ref_26 + properties: &ref_28 workspace_id: type: string name: @@ -4083,7 +4161,7 @@ paths: format: date-time format_extension: type: string - required: &ref_27 + required: &ref_29 - name responses: '201': @@ -4123,7 +4201,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: resource_type deleted @@ -4145,7 +4223,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 requestBody: description: updated resource_type required: true @@ -4153,7 +4231,7 @@ paths: application/json: schema: type: object - properties: &ref_167 + properties: &ref_169 schema: {} description: type: string @@ -4178,7 +4256,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: resource_type deleted @@ -4186,8 +4264,8 @@ paths: application/json: schema: type: object - properties: *ref_26 - required: *ref_27 + properties: *ref_28 + required: *ref_29 /w/{workspace}/resources/type/exists/{path}: get: summary: does resource_type exists @@ -4202,7 +4280,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: does resource_type exist @@ -4230,8 +4308,8 @@ paths: type: array items: type: object - properties: *ref_26 - required: *ref_27 + properties: *ref_28 + required: *ref_29 /w/{workspace}/resources/type/listnames: get: summary: list resource_types names @@ -4370,7 +4448,7 @@ paths: - name: id in: path required: true - schema: *ref_28 + schema: *ref_30 responses: '200': description: flow @@ -4381,43 +4459,32 @@ paths: properties: flow: type: object - properties: &ref_52 + properties: &ref_54 summary: type: string description: type: string value: type: object - properties: &ref_64 + properties: &ref_66 modules: type: array items: type: object - properties: &ref_31 + properties: &ref_33 id: type: string value: - oneOf: &ref_249 + oneOf: &ref_251 - type: object - properties: &ref_233 + properties: &ref_235 input_transforms: type: object additionalProperties: - oneOf: &ref_29 - - type: object - properties: &ref_229 - value: {} - type: - type: string - enum: - - javascript - required: &ref_230 - - expr - - type + oneOf: &ref_31 - type: object properties: &ref_231 - expr: - type: string + value: {} type: type: string enum: @@ -4425,7 +4492,18 @@ paths: required: &ref_232 - expr - type - discriminator: &ref_30 + - type: object + properties: &ref_233 + expr: + type: string + type: + type: string + enum: + - javascript + required: &ref_234 + - expr + - type + discriminator: &ref_32 propertyName: type mapping: static: '#/components/schemas/StaticTransform' @@ -4467,18 +4545,18 @@ paths: type: string is_trigger: type: boolean - required: &ref_234 + required: &ref_236 - type - content - language - input_transforms - type: object - properties: &ref_235 + properties: &ref_237 input_transforms: type: object additionalProperties: - oneOf: *ref_29 - discriminator: *ref_30 + oneOf: *ref_31 + discriminator: *ref_32 path: type: string hash: @@ -4491,63 +4569,40 @@ paths: type: string is_trigger: type: boolean - required: &ref_236 - - type - - path - - input_transforms - - type: object - properties: &ref_237 - input_transforms: - type: object - additionalProperties: - oneOf: *ref_29 - discriminator: *ref_30 - path: - type: string - type: - type: string - enum: - - flow required: &ref_238 - type - path - input_transforms - type: object properties: &ref_239 - modules: - type: array - items: - type: object - properties: *ref_31 - required: &ref_32 - - value - - id - iterator: - oneOf: *ref_29 - discriminator: *ref_30 - skip_failures: - type: boolean + input_transforms: + type: object + additionalProperties: + oneOf: *ref_31 + discriminator: *ref_32 + path: + type: string type: type: string enum: - - forloopflow - parallel: - type: boolean - parallelism: - type: integer + - flow required: &ref_240 - - modules - - iterator - - skip_failures - type + - path + - input_transforms - type: object properties: &ref_241 modules: type: array items: type: object - properties: *ref_31 - required: *ref_32 + properties: *ref_33 + required: &ref_34 + - value + - id + iterator: + oneOf: *ref_31 + discriminator: *ref_32 skip_failures: type: boolean type: @@ -4560,10 +4615,33 @@ paths: type: integer required: &ref_242 - modules + - iterator - skip_failures - type - type: object properties: &ref_243 + modules: + type: array + items: + type: object + properties: *ref_33 + required: *ref_34 + skip_failures: + type: boolean + type: + type: string + enum: + - forloopflow + parallel: + type: boolean + parallelism: + type: integer + required: &ref_244 + - modules + - skip_failures + - type + - type: object + properties: &ref_245 branches: type: array items: @@ -4577,8 +4655,8 @@ paths: type: array items: type: object - properties: *ref_31 - required: *ref_32 + properties: *ref_33 + required: *ref_34 required: - modules - expr @@ -4586,20 +4664,20 @@ paths: type: array items: type: object - properties: *ref_31 - required: *ref_32 + properties: *ref_33 + required: *ref_34 required: - modules type: type: string enum: - branchone - required: &ref_244 + required: &ref_246 - branches - default - type - type: object - properties: &ref_245 + properties: &ref_247 branches: type: array items: @@ -4613,8 +4691,8 @@ paths: type: array items: type: object - properties: *ref_31 - required: *ref_32 + properties: *ref_33 + required: *ref_34 required: - modules - expr @@ -4624,20 +4702,20 @@ paths: - branchall parallel: type: boolean - required: &ref_246 + required: &ref_248 - branches - type - type: object - properties: &ref_247 + properties: &ref_249 type: type: string enum: - identity flow: type: boolean - required: &ref_248 + required: &ref_250 - type - discriminator: &ref_250 + discriminator: &ref_252 propertyName: type mapping: rawscript: '#/components/schemas/RawScript' @@ -4674,8 +4752,8 @@ paths: required: - expr sleep: - oneOf: *ref_29 - discriminator: *ref_30 + oneOf: *ref_31 + discriminator: *ref_32 cache_ttl: type: number timeout: @@ -4705,8 +4783,8 @@ paths: user_auth_required: type: boolean user_groups_required: - oneOf: *ref_29 - discriminator: *ref_30 + oneOf: *ref_31 + discriminator: *ref_32 self_approval_disabled: type: boolean hide_cancel: @@ -4719,7 +4797,7 @@ paths: type: boolean retry: type: object - properties: &ref_93 + properties: &ref_95 constant: type: object properties: @@ -4740,15 +4818,15 @@ paths: type: integer minimum: 0 maximum: 100 - required: *ref_32 + required: *ref_34 failure_module: type: object - properties: *ref_31 - required: *ref_32 + properties: *ref_33 + required: *ref_34 preprocessor_module: type: object - properties: *ref_31 - required: *ref_32 + properties: *ref_33 + required: *ref_34 same_worker: type: boolean concurrent_limit: @@ -4765,11 +4843,11 @@ paths: type: number early_return: type: string - required: &ref_65 + required: &ref_67 - modules schema: type: object - required: &ref_53 + required: &ref_55 - summary - value /apps/hub/list: @@ -4822,7 +4900,7 @@ paths: - name: id in: path required: true - schema: *ref_28 + schema: *ref_30 responses: '200': description: app @@ -4852,7 +4930,7 @@ paths: - name: path in: path required: true - schema: &ref_33 + schema: &ref_35 type: string responses: '200': @@ -4871,7 +4949,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: script details @@ -4942,7 +5020,7 @@ paths: type: number kind: name: kind - schema: &ref_34 + schema: &ref_36 type: string enum: - script @@ -5015,7 +5093,7 @@ paths: type: string kind: name: kind - schema: *ref_34 + schema: *ref_36 score: type: number required: @@ -5076,12 +5154,12 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: &ref_50 + schema: &ref_52 type: boolean - name: created_by description: mask to filter exact matching user creator in: query - schema: &ref_51 + schema: &ref_53 type: string - name: path_start description: mask to filter matching starting path @@ -5195,7 +5273,7 @@ paths: type: array items: type: object - properties: &ref_35 + properties: &ref_37 workspace_id: type: string hash: @@ -5303,7 +5381,7 @@ paths: type: string has_preprocessor: type: boolean - required: &ref_36 + required: &ref_38 - hash - path - summary @@ -5401,7 +5479,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: draft deleted @@ -5427,7 +5505,7 @@ paths: application/json: schema: type: object - properties: &ref_40 + properties: &ref_42 path: type: string parent_hash: @@ -5509,7 +5587,7 @@ paths: type: string has_preprocessor: type: boolean - required: &ref_41 + required: &ref_43 - path - summary - description @@ -5536,7 +5614,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 requestBody: description: Workspace error handler enabled required: true @@ -5613,7 +5691,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: script archived @@ -5635,7 +5713,7 @@ paths: - name: hash in: path required: true - schema: &ref_37 + schema: &ref_39 type: string responses: '200': @@ -5644,8 +5722,8 @@ paths: application/json: schema: type: object - properties: *ref_35 - required: *ref_36 + properties: *ref_37 + required: *ref_38 /w/{workspace}/scripts/delete/h/{hash}: post: summary: delete script by hash (erase content but keep hash, require admin) @@ -5660,7 +5738,7 @@ paths: - name: hash in: path required: true - schema: *ref_37 + schema: *ref_39 responses: '200': description: script details @@ -5668,8 +5746,8 @@ paths: application/json: schema: type: object - properties: *ref_35 - required: *ref_36 + properties: *ref_37 + required: *ref_38 /w/{workspace}/scripts/delete/p/{path}: post: summary: delete all scripts at a given path (require admin) @@ -5684,7 +5762,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: script path @@ -5706,7 +5784,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 - name: with_starred_info in: query schema: @@ -5718,8 +5796,8 @@ paths: application/json: schema: type: object - properties: *ref_35 - required: *ref_36 + properties: *ref_37 + required: *ref_38 /w/{workspace}/scripts/get_triggers_count/{path}: get: summary: get triggers count of script @@ -5734,7 +5812,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: triggers count @@ -5742,7 +5820,7 @@ paths: application/json: schema: type: object - properties: &ref_57 + properties: &ref_59 primary_schedule: type: object properties: @@ -5772,7 +5850,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: tokens list @@ -5782,8 +5860,8 @@ paths: type: array items: type: object - properties: *ref_38 - required: *ref_39 + properties: *ref_40 + required: *ref_41 /w/{workspace}/scripts/get/draft/{path}: get: summary: get script by path with draft @@ -5798,23 +5876,23 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: script details content: application/json: schema: - allOf: &ref_135 + allOf: &ref_137 - type: object - properties: *ref_40 - required: *ref_41 + properties: *ref_42 + required: *ref_43 - type: object properties: draft: type: object - properties: *ref_40 - required: *ref_41 + properties: *ref_42 + required: *ref_43 hash: type: string required: @@ -5833,7 +5911,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: script history @@ -5843,12 +5921,12 @@ paths: type: array items: type: object - properties: &ref_42 + properties: &ref_44 script_hash: type: string deployment_msg: type: string - required: &ref_43 + required: &ref_45 - script_hash /w/{workspace}/scripts/get_latest_version/{path}: get: @@ -5862,7 +5940,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 tags: - script responses: @@ -5873,8 +5951,8 @@ paths: required: false schema: type: object - properties: *ref_42 - required: *ref_43 + properties: *ref_44 + required: *ref_45 /w/{workspace}/scripts/history_update/h/{hash}/p/{path}: post: summary: update history of a script @@ -5889,11 +5967,11 @@ paths: - name: hash in: path required: true - schema: *ref_37 + schema: *ref_39 - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 requestBody: description: Script deployment message required: true @@ -5925,7 +6003,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: script content @@ -5949,12 +6027,12 @@ paths: - name: token in: path required: true - schema: &ref_124 + schema: &ref_126 type: string - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: script content @@ -5976,7 +6054,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: does it exists @@ -5998,7 +6076,7 @@ paths: - name: hash in: path required: true - schema: *ref_37 + schema: *ref_39 - name: with_starred_info in: query schema: @@ -6010,8 +6088,8 @@ paths: application/json: schema: type: object - properties: *ref_35 - required: *ref_36 + properties: *ref_37 + required: *ref_38 /w/{workspace}/scripts/raw/h/{path}: get: summary: raw script by hash @@ -6026,7 +6104,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: script content @@ -6048,7 +6126,7 @@ paths: - name: hash in: path required: true - schema: *ref_37 + schema: *ref_39 responses: '200': description: script details @@ -6075,7 +6153,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -6097,20 +6175,20 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: &ref_44 + schema: &ref_46 type: string format: uuid - name: tag description: Override the tag to use in: query - schema: &ref_46 + schema: &ref_48 type: string - name: cache_ttl description: >- Override the cache time to live (in seconds). Can not be used to disable caching, only override with a new cache ttl in: query - schema: &ref_47 + schema: &ref_49 type: string - name: job_id description: >- @@ -6119,7 +6197,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: &ref_45 + schema: &ref_47 type: string format: uuid - name: invisible_to_owner @@ -6134,7 +6212,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 responses: '201': description: job created @@ -6157,13 +6235,13 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6171,7 +6249,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_45 + schema: *ref_47 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6180,14 +6258,14 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: &ref_48 + schema: &ref_50 type: string - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: &ref_49 + schema: &ref_51 type: string requestBody: description: script args @@ -6196,7 +6274,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 responses: '200': description: job result @@ -6217,23 +6295,23 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 - name: tag description: Override the tag to use in: query - schema: *ref_46 + schema: *ref_48 - name: cache_ttl description: >- Override the cache time to live (in seconds). Can not be used to disable caching, only override with a new cache ttl in: query - schema: *ref_47 + schema: *ref_49 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6241,7 +6319,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_45 + schema: *ref_47 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6250,13 +6328,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_48 + schema: *ref_50 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_49 + schema: *ref_51 requestBody: description: script args required: true @@ -6264,7 +6342,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 responses: '200': description: job result @@ -6284,23 +6362,23 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 - name: tag description: Override the tag to use in: query - schema: *ref_46 + schema: *ref_48 - name: cache_ttl description: >- Override the cache time to live (in seconds). Can not be used to disable caching, only override with a new cache ttl in: query - schema: *ref_47 + schema: *ref_49 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6308,7 +6386,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_45 + schema: *ref_47 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6317,13 +6395,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_48 + schema: *ref_50 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_49 + schema: *ref_51 - name: payload description: > The base64 encoded payload that has been encoded as a JSON. e.g how @@ -6331,7 +6409,7 @@ paths: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: &ref_92 + schema: &ref_94 type: string responses: '200': @@ -6353,7 +6431,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6362,13 +6440,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_48 + schema: *ref_50 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_49 + schema: *ref_51 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6376,7 +6454,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_45 + schema: *ref_47 requestBody: description: script args required: true @@ -6384,7 +6462,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 responses: '200': description: job result @@ -6405,7 +6483,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6414,13 +6492,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_48 + schema: *ref_50 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_49 + schema: *ref_51 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6428,7 +6506,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_45 + schema: *ref_47 requestBody: description: script args required: true @@ -6436,7 +6514,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 responses: '200': description: job result @@ -6539,11 +6617,11 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_50 + schema: *ref_52 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_51 + schema: *ref_53 - name: path_start description: mask to filter matching starting path in: query @@ -6597,12 +6675,12 @@ paths: type: array items: allOf: - - allOf: &ref_56 + - allOf: &ref_58 - type: object - properties: *ref_52 - required: *ref_53 + properties: *ref_54 + required: *ref_55 - type: object - properties: &ref_188 + properties: &ref_190 workspace_id: type: string path: @@ -6616,7 +6694,7 @@ paths: type: boolean extra_perms: type: object - additionalProperties: &ref_187 + additionalProperties: &ref_189 type: boolean starred: type: boolean @@ -6634,7 +6712,7 @@ paths: type: number visible_to_runner_only: type: boolean - required: &ref_189 + required: &ref_191 - path - edited_by - edited_at @@ -6658,7 +6736,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 tags: - flow responses: @@ -6670,7 +6748,7 @@ paths: type: array items: type: object - properties: &ref_54 + properties: &ref_56 id: type: integer created_at: @@ -6678,7 +6756,7 @@ paths: format: date-time deployment_msg: type: string - required: &ref_55 + required: &ref_57 - id - created_at /w/{workspace}/flows/get_latest_version/{path}: @@ -6693,7 +6771,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 tags: - flow responses: @@ -6704,8 +6782,8 @@ paths: required: false schema: type: object - properties: *ref_54 - required: *ref_55 + properties: *ref_56 + required: *ref_57 /w/{workspace}/flows/get/v/{version}/p/{path}: get: summary: get flow version @@ -6724,7 +6802,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 tags: - flow responses: @@ -6733,7 +6811,7 @@ paths: content: application/json: schema: - allOf: *ref_56 + allOf: *ref_58 /w/{workspace}/flows/history_update/v/{version}/p/{path}: post: summary: update flow history @@ -6752,7 +6830,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 requestBody: description: Flow deployment message required: true @@ -6788,7 +6866,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 - name: with_starred_info in: query schema: @@ -6799,7 +6877,7 @@ paths: content: application/json: schema: - allOf: *ref_56 + allOf: *ref_58 /w/{workspace}/flows/get_triggers_count/{path}: get: summary: get triggers count of flow @@ -6814,7 +6892,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: triggers count @@ -6822,7 +6900,7 @@ paths: application/json: schema: type: object - properties: *ref_57 + properties: *ref_59 /w/{workspace}/flows/list_tokens/{path}: get: summary: get tokens with flow scope @@ -6837,7 +6915,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: tokens list @@ -6847,8 +6925,8 @@ paths: type: array items: type: object - properties: *ref_38 - required: *ref_39 + properties: *ref_40 + required: *ref_41 /w/{workspace}/flows/toggle_workspace_error_handler/{path}: post: summary: Toggle ON and OFF the workspace error handler for a given flow @@ -6863,7 +6941,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 requestBody: description: Workspace error handler enabled required: true @@ -6895,7 +6973,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: flow details with draft @@ -6903,11 +6981,11 @@ paths: application/json: schema: allOf: - - allOf: *ref_56 + - allOf: *ref_58 - type: object properties: draft: - allOf: *ref_56 + allOf: *ref_58 /w/{workspace}/flows/exists/{path}: get: summary: exists flow by path @@ -6922,7 +7000,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: flow details @@ -6948,10 +7026,10 @@ paths: application/json: schema: allOf: - - allOf: &ref_58 + - allOf: &ref_60 - type: object - properties: *ref_52 - required: *ref_53 + properties: *ref_54 + required: *ref_55 - type: object properties: path: @@ -6997,7 +7075,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 requestBody: description: Partially filled flow required: true @@ -7005,7 +7083,7 @@ paths: application/json: schema: allOf: - - allOf: *ref_58 + - allOf: *ref_60 - type: object properties: deployment_message: @@ -7031,7 +7109,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 requestBody: description: archiveFlow required: true @@ -7063,7 +7141,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: flow delete @@ -7093,11 +7171,11 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_50 + schema: *ref_52 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_51 + schema: *ref_53 - name: path_start description: mask to filter matching starting path in: query @@ -7124,7 +7202,7 @@ paths: type: array items: type: object - properties: &ref_195 + properties: &ref_197 workspace_id: type: string path: @@ -7142,7 +7220,7 @@ paths: edited_at: type: string format: date-time - required: &ref_196 + required: &ref_198 - workspace_id - path - summary @@ -7163,7 +7241,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: app exists @@ -7185,12 +7263,12 @@ paths: - name: version in: path required: true - schema: &ref_123 + schema: &ref_125 type: number - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: app details @@ -7247,11 +7325,11 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_50 + schema: *ref_52 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_51 + schema: *ref_53 - name: path_start description: mask to filter matching starting path in: query @@ -7292,7 +7370,7 @@ paths: type: array items: type: object - properties: &ref_193 + properties: &ref_195 id: type: integer workspace_id: @@ -7318,7 +7396,7 @@ paths: - viewer - publisher - anonymous - required: &ref_194 + required: &ref_196 - id - workspace_id - path @@ -7353,7 +7431,7 @@ paths: type: string policy: type: object - properties: &ref_59 + properties: &ref_61 triggerables: type: object additionalProperties: @@ -7362,6 +7440,10 @@ paths: type: object additionalProperties: type: object + s3_inputs: + type: array + items: + type: object execution_mode: type: string enum: @@ -7402,7 +7484,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: app exists @@ -7424,7 +7506,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 - name: with_starred_info in: query schema: @@ -7436,7 +7518,7 @@ paths: application/json: schema: type: object - properties: &ref_60 + properties: &ref_62 id: type: integer workspace_id: @@ -7458,7 +7540,7 @@ paths: type: object policy: type: object - properties: *ref_59 + properties: *ref_61 execution_mode: type: string enum: @@ -7469,7 +7551,7 @@ paths: type: object additionalProperties: type: boolean - required: &ref_61 + required: &ref_63 - id - workspace_id - path @@ -7495,17 +7577,17 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: app details with draft content: application/json: schema: - allOf: &ref_197 + allOf: &ref_199 - type: object - properties: *ref_60 - required: *ref_61 + properties: *ref_62 + required: *ref_63 - type: object properties: draft_only: @@ -7525,7 +7607,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 responses: '200': description: app history @@ -7535,12 +7617,12 @@ paths: type: array items: type: object - properties: &ref_62 + properties: &ref_64 version: type: integer deployment_msg: type: string - required: &ref_63 + required: &ref_65 - version /w/{workspace}/apps/get_latest_version/{path}: get: @@ -7554,7 +7636,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 tags: - app responses: @@ -7565,8 +7647,8 @@ paths: required: false schema: type: object - properties: *ref_62 - required: *ref_63 + properties: *ref_64 + required: *ref_65 /w/{workspace}/apps/history_update/a/{id}/v/{version}: post: summary: update app history @@ -7581,11 +7663,11 @@ paths: - name: id in: path required: true - schema: *ref_28 + schema: *ref_30 - name: version in: path required: true - schema: &ref_125 + schema: &ref_127 type: integer requestBody: description: App deployment message @@ -7618,7 +7700,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: app details @@ -7626,8 +7708,8 @@ paths: application/json: schema: type: object - properties: *ref_60 - required: *ref_61 + properties: *ref_62 + required: *ref_63 /w/{workspace}/apps_u/public_resource/{path}: get: summary: get public resource @@ -7642,7 +7724,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: resource value @@ -7663,7 +7745,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: app secret @@ -7685,7 +7767,7 @@ paths: - name: id in: path required: true - schema: *ref_28 + schema: *ref_30 responses: '200': description: app details @@ -7693,8 +7775,8 @@ paths: application/json: schema: type: object - properties: *ref_60 - required: *ref_61 + properties: *ref_62 + required: *ref_63 /w/{workspace}/raw_apps/create: post: summary: create raw app @@ -7745,7 +7827,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 requestBody: description: updateraw app required: true @@ -7781,7 +7863,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: app deleted @@ -7803,7 +7885,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: app deleted @@ -7825,7 +7907,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 requestBody: description: update app required: true @@ -7841,7 +7923,7 @@ paths: value: {} policy: type: object - properties: *ref_59 + properties: *ref_61 deployment_message: type: string responses: @@ -7865,7 +7947,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 requestBody: description: update app required: true @@ -7927,7 +8009,7 @@ paths: - name: path in: path required: true - schema: *ref_33 + schema: *ref_35 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -7949,11 +8031,11 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 - name: tag description: Override the tag to use in: query - schema: *ref_46 + schema: *ref_48 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -7961,7 +8043,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_45 + schema: *ref_47 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -7970,7 +8052,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_48 + schema: *ref_50 - name: invisible_to_owner description: make the run invisible to the the flow owner (default false) in: query @@ -7983,7 +8065,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 responses: '201': description: job created @@ -8006,7 +8088,7 @@ paths: - name: id in: path required: true - schema: &ref_89 + schema: &ref_91 type: string format: uuid - name: step_id @@ -8039,11 +8121,11 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 - name: tag description: Override the tag to use in: query - schema: *ref_46 + schema: *ref_48 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -8051,7 +8133,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_45 + schema: *ref_47 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -8060,7 +8142,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_48 + schema: *ref_50 - name: invisible_to_owner description: make the run invisible to the the flow owner (default false) in: query @@ -8073,7 +8155,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 responses: '201': description: job created @@ -8096,7 +8178,7 @@ paths: - name: hash in: path required: true - schema: *ref_37 + schema: *ref_39 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -8118,17 +8200,17 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 - name: tag description: Override the tag to use in: query - schema: *ref_46 + schema: *ref_48 - name: cache_ttl description: >- Override the cache time to live (in seconds). Can not be used to disable caching, only override with a new cache ttl in: query - schema: *ref_47 + schema: *ref_49 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -8136,7 +8218,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_45 + schema: *ref_47 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -8145,7 +8227,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_48 + schema: *ref_50 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -8185,7 +8267,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_48 + schema: *ref_50 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -8198,7 +8280,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_45 + schema: *ref_47 requestBody: description: preview required: true @@ -8206,14 +8288,14 @@ paths: application/json: schema: type: object - properties: &ref_154 + properties: &ref_156 content: type: string path: type: string args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 language: type: string enum: @@ -8245,7 +8327,7 @@ paths: type: boolean lock: type: string - required: &ref_155 + required: &ref_157 - args responses: '201': @@ -8283,11 +8365,11 @@ paths: application/json: schema: type: object - properties: &ref_156 + properties: &ref_158 args: type: object - additionalProperties: *ref_16 - required: &ref_157 + additionalProperties: *ref_18 + required: &ref_159 - args responses: '201': @@ -8320,7 +8402,7 @@ paths: type: array items: type: object - properties: &ref_215 + properties: &ref_217 raw_code: type: string path: @@ -8344,7 +8426,7 @@ paths: - php - rust - ansible - required: &ref_216 + required: &ref_218 - raw_code - path - language @@ -8384,7 +8466,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_48 + schema: *ref_50 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -8397,7 +8479,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_45 + schema: *ref_47 requestBody: description: preview required: true @@ -8405,21 +8487,21 @@ paths: application/json: schema: type: object - properties: &ref_190 + properties: &ref_192 value: type: object - properties: *ref_64 - required: *ref_65 + properties: *ref_66 + required: *ref_67 path: type: string args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 tag: type: string restarted_from: type: object - properties: &ref_192 + properties: &ref_194 flow_job_id: type: string format: uuid @@ -8427,7 +8509,7 @@ paths: type: string branch_or_iteration_n: type: integer - required: &ref_191 + required: &ref_193 - value - content - args @@ -8453,94 +8535,94 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_50 + schema: *ref_52 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_51 + schema: *ref_53 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 - name: script_path_exact description: mask to filter exact matching path in: query - schema: &ref_68 + schema: &ref_70 type: string - name: script_path_start description: mask to filter matching starting path in: query - schema: &ref_69 + schema: &ref_71 type: string - name: schedule_path description: mask to filter by schedule path in: query - schema: &ref_70 + schema: &ref_72 type: string - name: script_hash description: mask to filter exact matching path in: query - schema: &ref_71 + schema: &ref_73 type: string - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: &ref_72 + schema: &ref_74 type: string format: date-time - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: &ref_73 + schema: &ref_75 type: string format: date-time - name: success description: filter on successful jobs in: query - schema: &ref_74 + schema: &ref_76 type: boolean - name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: &ref_75 + schema: &ref_77 type: boolean - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: &ref_76 + schema: &ref_78 type: string - name: suspended description: filter on suspended jobs in: query - schema: &ref_77 + schema: &ref_79 type: boolean - name: running description: filter on running jobs in: query - schema: &ref_78 + schema: &ref_80 type: boolean - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: &ref_79 + schema: &ref_81 type: string - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: &ref_80 + schema: &ref_82 type: string - name: tag description: filter on jobs with a given tag/worker group in: query - schema: &ref_81 + schema: &ref_83 type: string - name: page description: which page to return (start at 1, default 1) @@ -8571,7 +8653,7 @@ paths: type: array items: type: object - properties: &ref_87 + properties: &ref_89 workspace_id: type: string id: @@ -8599,7 +8681,7 @@ paths: type: string args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 logs: type: string raw_code: @@ -8638,14 +8720,14 @@ paths: by extension its DT_TOKEN. flow_status: type: object - properties: &ref_82 + properties: &ref_84 step: type: integer modules: type: array items: type: object - properties: &ref_66 + properties: &ref_68 type: type: string enum: @@ -8722,20 +8804,20 @@ paths: format: uuid skipped: type: boolean - required: &ref_67 + required: &ref_69 - type user_states: additionalProperties: true preprocessor_module: allOf: - type: object - properties: *ref_66 - required: *ref_67 + properties: *ref_68 + required: *ref_69 failure_module: allOf: - type: object - properties: *ref_66 - required: *ref_67 + properties: *ref_68 + required: *ref_69 - type: object properties: parent_module: @@ -8750,14 +8832,14 @@ paths: items: type: string format: uuid - required: &ref_83 + required: &ref_85 - step - modules - failure_module raw_flow: type: object - properties: *ref_64 - required: *ref_65 + properties: *ref_66 + required: *ref_67 is_flow_step: type: boolean language: @@ -8795,7 +8877,7 @@ paths: type: number suspend: type: number - required: &ref_88 + required: &ref_90 - id - running - canceled @@ -8874,79 +8956,79 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_50 + schema: *ref_52 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_51 + schema: *ref_53 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_68 + schema: *ref_70 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_69 + schema: *ref_71 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_70 + schema: *ref_72 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_71 + schema: *ref_73 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_72 + schema: *ref_74 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_73 + schema: *ref_75 - name: success description: filter on successful jobs in: query - schema: *ref_74 + schema: *ref_76 - name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: *ref_75 + schema: *ref_77 - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_76 + schema: *ref_78 - name: suspended description: filter on suspended jobs in: query - schema: *ref_77 + schema: *ref_79 - name: running description: filter on running jobs in: query - schema: *ref_78 + schema: *ref_80 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_79 + schema: *ref_81 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_80 + schema: *ref_82 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_81 + schema: *ref_83 - name: page description: which page to return (start at 1, default 1) in: query @@ -9024,75 +9106,75 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_50 + schema: *ref_52 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_51 + schema: *ref_53 - name: label description: >- mask to filter exact matching job's label (job labels are completed jobs with as a result an object containing a string in the array at key 'wm_labels') in: query - schema: &ref_84 + schema: &ref_86 type: string - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_68 + schema: *ref_70 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_69 + schema: *ref_71 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_70 + schema: *ref_72 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_71 + schema: *ref_73 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_72 + schema: *ref_74 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_73 + schema: *ref_75 - name: success description: filter on successful jobs in: query - schema: *ref_74 + schema: *ref_76 - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_76 + schema: *ref_78 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_79 + schema: *ref_81 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_80 + schema: *ref_82 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_81 + schema: *ref_83 - name: page description: which page to return (start at 1, default 1) in: query @@ -9130,7 +9212,7 @@ paths: type: array items: type: object - properties: &ref_85 + properties: &ref_87 workspace_id: type: string id: @@ -9157,7 +9239,7 @@ paths: type: string args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 result: {} logs: type: string @@ -9196,12 +9278,12 @@ paths: by extension its DT_TOKEN. flow_status: type: object - properties: *ref_82 - required: *ref_83 + properties: *ref_84 + required: *ref_85 raw_flow: type: object - properties: *ref_64 - required: *ref_65 + properties: *ref_66 + required: *ref_67 is_flow_step: type: boolean language: @@ -9243,7 +9325,7 @@ paths: type: number aggregate_wait_time_ms: type: number - required: &ref_86 + required: &ref_88 - id - created_by - duration_ms @@ -9272,54 +9354,54 @@ paths: - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_51 + schema: *ref_53 - name: label description: >- mask to filter exact matching job's label (job labels are completed jobs with as a result an object containing a string in the array at key 'wm_labels') in: query - schema: *ref_84 + schema: *ref_86 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_68 + schema: *ref_70 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_69 + schema: *ref_71 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_70 + schema: *ref_72 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_71 + schema: *ref_73 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_72 + schema: *ref_74 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_73 + schema: *ref_75 - name: created_before description: filter on created before (inclusive) timestamp in: query - schema: &ref_126 + schema: &ref_128 type: string format: date-time - name: created_after description: filter on created after (exclusive) timestamp in: query - schema: &ref_127 + schema: &ref_129 type: string format: date-time - name: created_or_started_before @@ -9327,23 +9409,23 @@ paths: filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query - schema: &ref_120 + schema: &ref_122 type: string format: date-time - name: running description: filter on running jobs in: query - schema: *ref_78 + schema: *ref_80 - name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: *ref_75 + schema: *ref_77 - name: created_or_started_after description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query - schema: &ref_121 + schema: &ref_123 type: string format: date-time - name: created_or_started_after_completed_jobs @@ -9352,7 +9434,7 @@ paths: otherwise after (exclusive) timestamp but only for the completed jobs in: query - schema: &ref_122 + schema: &ref_124 type: string format: date-time - name: job_kinds @@ -9360,27 +9442,27 @@ paths: filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_76 + schema: *ref_78 - name: suspended description: filter on suspended jobs in: query - schema: *ref_77 + schema: *ref_79 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_79 + schema: *ref_81 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_81 + schema: *ref_83 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_80 + schema: *ref_82 - name: page description: which page to return (start at 1, default 1) in: query @@ -9429,28 +9511,28 @@ paths: schema: type: array items: - oneOf: &ref_90 - - allOf: - - type: object - properties: *ref_85 - required: *ref_86 - - type: object - properties: - type: - type: string - enum: - - CompletedJob + oneOf: &ref_92 - allOf: - type: object properties: *ref_87 required: *ref_88 + - type: object + properties: + type: + type: string + enum: + - CompletedJob + - allOf: + - type: object + properties: *ref_89 + required: *ref_90 - type: object properties: type: type: string enum: - QueuedJob - discriminator: &ref_91 + discriminator: &ref_93 propertyName: type /jobs/db_clock: get: @@ -9517,7 +9599,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: no_logs in: query schema: @@ -9528,8 +9610,8 @@ paths: content: application/json: schema: - oneOf: *ref_90 - discriminator: *ref_91 + oneOf: *ref_92 + discriminator: *ref_93 /w/{workspace}/jobs_u/get_root_job_id/{id}: get: summary: get root job id @@ -9544,7 +9626,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 responses: '200': description: get root job id @@ -9566,7 +9648,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 responses: '200': description: job details @@ -9588,7 +9670,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 responses: '200': description: job args @@ -9609,7 +9691,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: running in: query schema: @@ -9644,9 +9726,9 @@ paths: type: integer flow_status: type: object - additionalProperties: &ref_158 + additionalProperties: &ref_160 type: object - properties: &ref_159 + properties: &ref_161 scheduled_for: type: string format: date-time @@ -9693,7 +9775,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 responses: '200': description: flow debug info details @@ -9714,7 +9796,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 responses: '200': description: job details @@ -9722,8 +9804,8 @@ paths: application/json: schema: type: object - properties: *ref_85 - required: *ref_86 + properties: *ref_87 + required: *ref_88 /w/{workspace}/jobs_u/completed/get_result/{id}: get: summary: get completed job result @@ -9738,7 +9820,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: suspended_job in: query schema: @@ -9775,10 +9857,10 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: get_started in: query - schema: &ref_133 + schema: &ref_135 type: boolean responses: '200': @@ -9812,7 +9894,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 responses: '200': description: job details @@ -9820,8 +9902,8 @@ paths: application/json: schema: type: object - properties: *ref_85 - required: *ref_86 + properties: *ref_87 + required: *ref_88 /w/{workspace}/jobs_u/queue/cancel/{id}: post: summary: cancel queued or running job @@ -9836,7 +9918,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 requestBody: description: reason required: true @@ -9868,7 +9950,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 requestBody: description: reason required: true @@ -9900,7 +9982,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 requestBody: description: reason required: true @@ -9932,7 +10014,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: resume_id in: path required: true @@ -9963,7 +10045,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: resume_id in: path required: true @@ -10005,7 +10087,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: payload description: > The base64 encoded payload that has been encoded as a JSON. e.g how @@ -10013,7 +10095,7 @@ paths: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: *ref_92 + schema: *ref_94 - name: resume_id in: path required: true @@ -10048,7 +10130,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: resume_id in: path required: true @@ -10090,7 +10172,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: key in: path required: true @@ -10122,7 +10204,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: key in: path required: true @@ -10148,7 +10230,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 requestBody: required: true content: @@ -10176,7 +10258,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: resume_id in: path required: true @@ -10211,7 +10293,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: resume_id in: path required: true @@ -10253,7 +10335,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 - name: resume_id in: path required: true @@ -10277,8 +10359,8 @@ paths: type: object properties: job: - oneOf: *ref_90 - discriminator: *ref_91 + oneOf: *ref_92 + discriminator: *ref_93 approvers: type: array items: @@ -10343,7 +10425,7 @@ paths: application/json: schema: type: object - properties: &ref_169 + properties: &ref_171 path: type: string schedule: @@ -10356,7 +10438,7 @@ paths: type: boolean args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 enabled: type: boolean on_failure: @@ -10367,24 +10449,24 @@ paths: type: boolean on_failure_extra_args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 on_recovery: type: string on_recovery_times: type: number on_recovery_extra_args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 on_success: type: string on_success_extra_args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 ws_error_handler_muted: type: boolean retry: type: object - properties: *ref_93 + properties: *ref_95 no_flow_overlap: type: boolean summary: @@ -10394,7 +10476,7 @@ paths: paused_until: type: string format: date-time - required: &ref_170 + required: &ref_172 - path - schedule - timezone @@ -10422,7 +10504,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 requestBody: description: updated schedule required: true @@ -10430,14 +10512,14 @@ paths: application/json: schema: type: object - properties: &ref_171 + properties: &ref_173 schedule: type: string timezone: type: string args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 on_failure: type: string on_failure_times: @@ -10446,24 +10528,24 @@ paths: type: boolean on_failure_extra_args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 on_recovery: type: string on_recovery_times: type: number on_recovery_extra_args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 on_success: type: string on_success_extra_args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 ws_error_handler_muted: type: boolean retry: type: object - properties: *ref_93 + properties: *ref_95 no_flow_overlap: type: boolean summary: @@ -10473,7 +10555,7 @@ paths: paused_until: type: string format: date-time - required: &ref_172 + required: &ref_174 - schedule - timezone - script_path @@ -10500,7 +10582,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 requestBody: description: updated schedule enable required: true @@ -10534,7 +10616,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: schedule deleted @@ -10556,7 +10638,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: schedule deleted @@ -10564,7 +10646,7 @@ paths: application/json: schema: type: object - properties: &ref_94 + properties: &ref_96 path: type: string edited_by: @@ -10584,7 +10666,7 @@ paths: type: boolean args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 extra_perms: type: object additionalProperties: @@ -10601,24 +10683,24 @@ paths: type: boolean on_failure_extra_args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 on_recovery: type: string on_recovery_times: type: number on_recovery_extra_args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 on_success: type: string on_success_extra_args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 ws_error_handler_muted: type: boolean retry: type: object - properties: *ref_93 + properties: *ref_95 summary: type: string no_flow_overlap: @@ -10628,7 +10710,7 @@ paths: paused_until: type: string format: date-time - required: &ref_95 + required: &ref_97 - path - edited_by - edited_at @@ -10653,7 +10735,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: schedule exists @@ -10685,7 +10767,7 @@ paths: filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_79 + schema: *ref_81 - name: path description: filter by path in: query @@ -10708,8 +10790,8 @@ paths: type: array items: type: object - properties: *ref_94 - required: *ref_95 + properties: *ref_96 + required: *ref_97 /w/{workspace}/schedules/list_with_jobs: get: summary: list schedules with last 20 jobs @@ -10737,10 +10819,10 @@ paths: schema: type: array items: - allOf: &ref_168 + allOf: &ref_170 - type: object - properties: *ref_94 - required: *ref_95 + properties: *ref_96 + required: *ref_97 - type: object properties: jobs: @@ -10819,13 +10901,24 @@ paths: application/json: schema: type: object - properties: &ref_173 + properties: &ref_175 path: type: string script_path: type: string route_path: type: string + static_asset_config: + type: object + properties: + s3: + type: string + storage: + type: string + filename: + type: string + required: + - s3 is_flow: type: boolean http_method: @@ -10840,7 +10933,7 @@ paths: type: boolean requires_auth: type: boolean - required: &ref_174 + required: &ref_176 - path - script_path - route_path @@ -10869,7 +10962,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 requestBody: description: updated trigger required: true @@ -10877,13 +10970,24 @@ paths: application/json: schema: type: object - properties: &ref_175 + properties: &ref_177 path: type: string script_path: type: string route_path: type: string + static_asset_config: + type: object + properties: + s3: + type: string + storage: + type: string + filename: + type: string + required: + - s3 is_flow: type: boolean http_method: @@ -10898,7 +11002,7 @@ paths: type: boolean requires_auth: type: boolean - required: &ref_176 + required: &ref_178 - path - script_path - is_flow @@ -10927,7 +11031,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: http trigger deleted @@ -10949,7 +11053,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: http trigger deleted @@ -10957,7 +11061,7 @@ paths: application/json: schema: type: object - properties: &ref_96 + properties: &ref_98 path: type: string edited_by: @@ -10969,6 +11073,17 @@ paths: type: string route_path: type: string + static_asset_config: + type: object + properties: + s3: + type: string + storage: + type: string + filename: + type: string + required: + - s3 is_flow: type: boolean extra_perms: @@ -10991,7 +11106,7 @@ paths: type: boolean requires_auth: type: boolean - required: &ref_97 + required: &ref_99 - path - edited_by - edited_at @@ -11045,8 +11160,8 @@ paths: type: array items: type: object - properties: *ref_96 - required: *ref_97 + properties: *ref_98 + required: *ref_99 /w/{workspace}/http_triggers/exists/{path}: get: summary: does http trigger exists @@ -11061,7 +11176,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: http trigger exists @@ -11127,7 +11242,7 @@ paths: application/json: schema: type: object - properties: &ref_177 + properties: &ref_179 path: type: string script_path: @@ -11152,7 +11267,7 @@ paths: initial_messages: type: array items: - anyOf: &ref_98 + anyOf: &ref_100 - type: object properties: raw_message: @@ -11168,7 +11283,7 @@ paths: type: string args: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 is_flow: type: boolean required: @@ -11179,8 +11294,8 @@ paths: - runnable_result url_runnable_args: type: object - additionalProperties: *ref_16 - required: &ref_178 + additionalProperties: *ref_18 + required: &ref_180 - path - script_path - url @@ -11209,7 +11324,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 requestBody: description: updated trigger required: true @@ -11217,7 +11332,7 @@ paths: application/json: schema: type: object - properties: &ref_179 + properties: &ref_181 url: type: string path: @@ -11240,11 +11355,11 @@ paths: initial_messages: type: array items: - anyOf: *ref_98 + anyOf: *ref_100 url_runnable_args: type: object - additionalProperties: *ref_16 - required: &ref_180 + additionalProperties: *ref_18 + required: &ref_182 - path - script_path - url @@ -11273,7 +11388,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: websocket trigger deleted @@ -11295,7 +11410,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: websocket trigger deleted @@ -11303,7 +11418,7 @@ paths: application/json: schema: type: object - properties: &ref_99 + properties: &ref_101 path: type: string edited_by: @@ -11348,11 +11463,11 @@ paths: initial_messages: type: array items: - anyOf: *ref_98 + anyOf: *ref_100 url_runnable_args: type: object - additionalProperties: *ref_16 - required: &ref_100 + additionalProperties: *ref_18 + required: &ref_102 - path - edited_by - edited_at @@ -11407,8 +11522,8 @@ paths: type: array items: type: object - properties: *ref_99 - required: *ref_100 + properties: *ref_101 + required: *ref_102 /w/{workspace}/websocket_triggers/exists/{path}: get: summary: does websocket trigger exists @@ -11423,7 +11538,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: websocket trigger exists @@ -11445,7 +11560,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 requestBody: description: updated websocket trigger enable required: true @@ -11480,7 +11595,7 @@ paths: type: array items: type: object - properties: &ref_102 + properties: &ref_104 name: type: string summary: @@ -11489,7 +11604,7 @@ paths: type: array items: type: string - required: &ref_103 + required: &ref_105 - name /groups/get/{name}: get: @@ -11501,7 +11616,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 responses: '200': description: instance group @@ -11509,8 +11624,8 @@ paths: application/json: schema: type: object - properties: *ref_102 - required: *ref_103 + properties: *ref_104 + required: *ref_105 /groups/create: post: summary: create instance group @@ -11548,7 +11663,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 requestBody: description: update instance group required: true @@ -11578,7 +11693,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 responses: '200': description: instance group deleted @@ -11596,7 +11711,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 requestBody: description: user to add to instance group required: true @@ -11626,7 +11741,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 requestBody: description: user to remove from instance group required: true @@ -11661,7 +11776,7 @@ paths: type: array items: type: object - properties: &ref_104 + properties: &ref_106 name: type: string summary: @@ -11676,7 +11791,7 @@ paths: type: string external_id: type: string - required: &ref_105 + required: &ref_107 - name /groups/overwrite: post: @@ -11693,8 +11808,8 @@ paths: type: array items: type: object - properties: *ref_104 - required: *ref_105 + properties: *ref_106 + required: *ref_107 responses: '200': description: success message @@ -11730,7 +11845,7 @@ paths: type: array items: type: object - properties: &ref_106 + properties: &ref_108 name: type: string summary: @@ -11743,7 +11858,7 @@ paths: type: object additionalProperties: type: boolean - required: &ref_107 + required: &ref_109 - name /w/{workspace}/groups/listnames: get: @@ -11816,7 +11931,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 requestBody: description: updated group required: true @@ -11848,7 +11963,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 responses: '200': description: group deleted @@ -11870,7 +11985,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 responses: '200': description: group @@ -11878,8 +11993,8 @@ paths: application/json: schema: type: object - properties: *ref_106 - required: *ref_107 + properties: *ref_108 + required: *ref_109 /w/{workspace}/groups/adduser/{name}: post: summary: add user to group @@ -11894,7 +12009,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 requestBody: description: added user to group required: true @@ -11926,7 +12041,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 requestBody: description: added user to group required: true @@ -11972,7 +12087,7 @@ paths: type: array items: type: object - properties: &ref_108 + properties: &ref_110 name: type: string owners: @@ -11990,7 +12105,7 @@ paths: edited_at: type: string format: date-time - required: &ref_109 + required: &ref_111 - name - owners - extra_perms @@ -12072,7 +12187,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 requestBody: description: update folder required: true @@ -12111,7 +12226,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 responses: '200': description: folder deleted @@ -12133,7 +12248,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 responses: '200': description: folder @@ -12141,8 +12256,8 @@ paths: application/json: schema: type: object - properties: *ref_108 - required: *ref_109 + properties: *ref_110 + required: *ref_111 /w/{workspace}/folders/getusage/{name}: get: summary: get folder usage @@ -12157,7 +12272,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 responses: '200': description: folder @@ -12199,7 +12314,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 requestBody: description: owner user to folder required: true @@ -12233,7 +12348,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 requestBody: description: added owner to folder required: true @@ -12287,7 +12402,7 @@ paths: type: array items: type: object - properties: &ref_181 + properties: &ref_183 worker: type: string worker_instance: @@ -12329,7 +12444,7 @@ paths: type: number wm_memory_usage: type: number - required: &ref_182 + required: &ref_184 - worker - worker_instance - ping_at @@ -12390,6 +12505,21 @@ paths: required: - id - values + /workers/queue_counts: + get: + summary: get counts of jobs waiting for an executor per tag + operationId: getCountsOfJobsWaitingPerTag + tags: + - worker + responses: + '200': + description: queue counts + content: + application/json: + schema: + type: object + additionalProperties: + type: integer /configs/list_worker_groups: get: summary: list worker groups @@ -12422,7 +12552,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 responses: '200': description: a config @@ -12439,7 +12569,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 requestBody: description: worker group required: true @@ -12462,7 +12592,7 @@ paths: - name: name in: path required: true - schema: *ref_101 + schema: *ref_103 responses: '200': description: Delete config @@ -12485,12 +12615,12 @@ paths: type: array items: type: object - properties: &ref_223 + properties: &ref_225 name: type: string config: type: object - required: &ref_224 + required: &ref_226 - name /configs/list_autoscaling_events/{worker_group}: get: @@ -12513,7 +12643,7 @@ paths: type: array items: type: object - properties: &ref_227 + properties: &ref_229 id: type: integer format: int64 @@ -12542,7 +12672,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 - name: kind in: path required: true @@ -12583,7 +12713,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 - name: kind in: path required: true @@ -12636,7 +12766,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 - name: kind in: path required: true @@ -12687,7 +12817,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '204': description: flow preview captured @@ -12705,7 +12835,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '201': description: flow preview capture created @@ -12722,7 +12852,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: captured flow preview @@ -12802,13 +12932,13 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: &ref_110 + schema: &ref_112 type: string - name: runnable_type in: query - schema: &ref_111 + schema: &ref_113 type: string - enum: &ref_140 + enum: &ref_142 - ScriptHash - ScriptPath - FlowPath @@ -12829,7 +12959,7 @@ paths: type: array items: type: object - properties: &ref_112 + properties: &ref_114 id: type: string name: @@ -12843,7 +12973,7 @@ paths: type: boolean success: type: boolean - required: &ref_113 + required: &ref_115 - id - name - args @@ -12893,10 +13023,10 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: *ref_110 + schema: *ref_112 - name: runnable_type in: query - schema: *ref_111 + schema: *ref_113 - name: page description: which page to return (start at 1, default 1) in: query @@ -12914,8 +13044,8 @@ paths: type: array items: type: object - properties: *ref_112 - required: *ref_113 + properties: *ref_114 + required: *ref_115 /w/{workspace}/inputs/create: post: summary: Create an Input for future use in a script or flow @@ -12929,10 +13059,10 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: *ref_110 + schema: *ref_112 - name: runnable_type in: query - schema: *ref_111 + schema: *ref_113 requestBody: description: Input required: true @@ -12940,12 +13070,12 @@ paths: application/json: schema: type: object - properties: &ref_136 + properties: &ref_138 name: type: string args: type: object - required: &ref_137 + required: &ref_139 - name - args - created_by @@ -12975,14 +13105,14 @@ paths: application/json: schema: type: object - properties: &ref_138 + properties: &ref_140 id: type: string name: type: string is_public: type: boolean - required: &ref_139 + required: &ref_141 - id - name - is_public @@ -13008,7 +13138,7 @@ paths: - name: input in: path required: true - schema: &ref_132 + schema: &ref_134 type: string responses: '200': @@ -13041,7 +13171,7 @@ paths: properties: s3_resource: type: object - properties: &ref_114 + properties: &ref_116 bucket: type: string region: @@ -13056,7 +13186,7 @@ paths: type: string pathStyle: type: boolean - required: &ref_115 + required: &ref_117 - bucket - region - endPoint @@ -13132,8 +13262,8 @@ paths: properties: s3_resource: type: object - properties: *ref_114 - required: *ref_115 + properties: *ref_116 + required: *ref_117 responses: '200': description: Connection settings @@ -13154,10 +13284,10 @@ paths: type: boolean client_kwargs: type: object - properties: &ref_116 + properties: &ref_118 region_name: type: string - required: &ref_117 + required: &ref_119 - region_name required: - endpoint_url @@ -13212,8 +13342,8 @@ paths: type: boolean client_kwargs: type: object - properties: *ref_116 - required: *ref_117 + properties: *ref_118 + required: *ref_119 required: - endpoint_url - use_ssl @@ -13271,8 +13401,8 @@ paths: application/json: schema: type: object - properties: *ref_114 - required: *ref_115 + properties: *ref_116 + required: *ref_117 /w/{workspace}/job_helpers/test_connection: get: summary: Test connection to the workspace object storage @@ -13336,10 +13466,10 @@ paths: type: array items: type: object - properties: &ref_200 + properties: &ref_202 s3: type: string - required: &ref_201 + required: &ref_203 - s3 restricted_access: type: boolean @@ -13372,7 +13502,7 @@ paths: application/json: schema: type: object - properties: &ref_202 + properties: &ref_204 mime_type: type: string size_in_bytes: @@ -13436,7 +13566,7 @@ paths: application/json: schema: type: object - properties: &ref_203 + properties: &ref_205 msg: type: string content: @@ -13448,7 +13578,7 @@ paths: - Csv - Parquet - Unknown - required: &ref_204 + required: &ref_206 - content_type /w/{workspace}/job_helpers/load_parquet_preview/{path}: get: @@ -13464,7 +13594,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 - name: offset in: query schema: @@ -13513,7 +13643,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 - name: search_col in: query schema: @@ -13550,7 +13680,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 - name: offset in: query schema: @@ -13681,6 +13811,14 @@ paths: in: query schema: type: string + - name: content_type + in: query + schema: + type: string + - name: content_disposition + in: query + schema: + type: string requestBody: description: File content required: true @@ -13786,7 +13924,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 requestBody: description: parameters for statistics retrieval required: true @@ -13815,46 +13953,46 @@ paths: type: array items: type: object - properties: &ref_207 + properties: &ref_209 id: type: string name: type: string - required: &ref_208 - - id - scalar_metrics: - type: array - items: - type: object - properties: &ref_209 - metric_id: - type: string - value: - type: number required: &ref_210 - id - - value - timeseries_metrics: + scalar_metrics: type: array items: type: object properties: &ref_211 + metric_id: + type: string + value: + type: number + required: &ref_212 + - id + - value + timeseries_metrics: + type: array + items: + type: object + properties: &ref_213 metric_id: type: string values: type: array items: type: object - properties: &ref_213 + properties: &ref_215 timestamp: type: string format: date-time value: type: number - required: &ref_214 + required: &ref_216 - timestamp - value - required: &ref_212 + required: &ref_214 - id - values /w/{workspace}/job_metrics/set_progress/{id}: @@ -13871,7 +14009,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 requestBody: description: parameters for statistics retrieval required: true @@ -13905,7 +14043,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 responses: '200': description: job progress between 0 and 99 @@ -13923,11 +14061,11 @@ paths: - name: before description: filter on started before (inclusive) timestamp in: query - schema: *ref_118 + schema: *ref_120 - name: after description: filter on created after (exclusive) timestamp in: query - schema: *ref_119 + schema: *ref_121 - name: with_error in: query required: false @@ -13976,7 +14114,7 @@ paths: - name: path in: path required: true - schema: *ref_21 + schema: *ref_23 responses: '200': description: log stream @@ -13999,12 +14137,12 @@ paths: type: array items: type: object - properties: &ref_217 + properties: &ref_219 concurrency_key: type: string total_running: type: number - required: &ref_218 + required: &ref_220 - concurrency_key - total_running /concurrency_groups/prune/{concurrency_id}: @@ -14017,7 +14155,7 @@ paths: - name: concurrency_id in: path required: true - schema: &ref_134 + schema: &ref_136 type: string responses: '200': @@ -14037,7 +14175,7 @@ paths: - name: id in: path required: true - schema: *ref_89 + schema: *ref_91 responses: '200': description: concurrency key for given job @@ -14070,93 +14208,93 @@ paths: - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_51 + schema: *ref_53 - name: label description: >- mask to filter exact matching job's label (job labels are completed jobs with as a result an object containing a string in the array at key 'wm_labels') in: query - schema: *ref_84 + schema: *ref_86 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_68 + schema: *ref_70 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_69 + schema: *ref_71 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_70 + schema: *ref_72 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_71 + schema: *ref_73 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_72 + schema: *ref_74 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_73 + schema: *ref_75 - name: created_or_started_before description: >- filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query - schema: *ref_120 + schema: *ref_122 - name: running description: filter on running jobs in: query - schema: *ref_78 + schema: *ref_80 - name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: *ref_75 + schema: *ref_77 - name: created_or_started_after description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query - schema: *ref_121 + schema: *ref_123 - name: created_or_started_after_completed_jobs description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp but only for the completed jobs in: query - schema: *ref_122 + schema: *ref_124 - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_76 + schema: *ref_78 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_79 + schema: *ref_81 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_81 + schema: *ref_83 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_80 + schema: *ref_82 - name: page description: which page to return (start at 1, default 1) in: query @@ -14204,17 +14342,17 @@ paths: application/json: schema: type: object - properties: &ref_219 + properties: &ref_221 jobs: type: array items: - oneOf: *ref_90 - discriminator: *ref_91 + oneOf: *ref_92 + discriminator: *ref_93 obscured_jobs: type: array items: type: object - properties: &ref_141 + properties: &ref_143 typ: type: string started_at: @@ -14227,7 +14365,7 @@ paths: Obscured jobs omitted for security because of too specific filtering type: boolean - required: &ref_220 + required: &ref_222 - jobs - obscured_jobs /srch/w/{workspace}/index/search/job: @@ -14269,7 +14407,7 @@ paths: type: array items: type: object - properties: &ref_225 + properties: &ref_227 dancer: type: string /srch/index/search/service_logs: @@ -14331,7 +14469,7 @@ paths: type: array items: type: object - properties: &ref_226 + properties: &ref_228 dancer: type: string /srch/index/search/count_service_logs: @@ -14405,57 +14543,57 @@ components: name: version in: path required: true - schema: *ref_123 + schema: *ref_125 Token: name: token in: path required: true - schema: *ref_124 + schema: *ref_126 AccountId: name: id in: path required: true - schema: *ref_25 + schema: *ref_27 ClientName: name: client_name in: path required: true - schema: *ref_24 + schema: *ref_26 ScriptPath: name: path in: path required: true - schema: *ref_33 + schema: *ref_35 ScriptHash: name: hash in: path required: true - schema: *ref_37 + schema: *ref_39 JobId: name: id in: path required: true - schema: *ref_89 + schema: *ref_91 Path: name: path in: path required: true - schema: *ref_21 + schema: *ref_23 PathId: name: id in: path required: true - schema: *ref_28 + schema: *ref_30 PathVersion: name: version in: path required: true - schema: *ref_125 + schema: *ref_127 Name: name: name in: path required: true - schema: *ref_101 + schema: *ref_103 Page: name: page description: which page to return (start at 1, default 1) @@ -14470,12 +14608,12 @@ components: name: order_desc description: order by desc order (default true) in: query - schema: *ref_50 + schema: *ref_52 CreatedBy: name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_51 + schema: *ref_53 Label: name: label description: >- @@ -14483,26 +14621,26 @@ components: with as a result an object containing a string in the array at key 'wm_labels') in: query - schema: *ref_84 + schema: *ref_86 ParentJob: name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_44 + schema: *ref_46 WorkerTag: name: tag description: Override the tag to use in: query - schema: *ref_46 + schema: *ref_48 CacheTtl: name: cache_ttl description: >- Override the cache time to live (in seconds). Can not be used to disable caching, only override with a new cache ttl in: query - schema: *ref_47 + schema: *ref_49 NewJobId: name: job_id description: >- @@ -14510,7 +14648,7 @@ components: randomly using the ULID scheme. If a job id already exists in the queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_45 + schema: *ref_47 IncludeHeader: name: include_header description: > @@ -14520,14 +14658,14 @@ components: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_48 + schema: *ref_50 QueueLimit: name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_49 + schema: *ref_51 Payload: name: payload description: > @@ -14536,249 +14674,253 @@ components: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: *ref_92 + schema: *ref_94 ScriptStartPath: name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_69 + schema: *ref_71 SchedulePath: name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_70 + schema: *ref_72 ScriptExactPath: name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_68 + schema: *ref_70 ScriptExactHash: name: script_hash description: mask to filter exact matching path in: query - schema: *ref_71 + schema: *ref_73 CreatedBefore: name: created_before description: filter on created before (inclusive) timestamp in: query - schema: *ref_126 + schema: *ref_128 CreatedAfter: name: created_after description: filter on created after (exclusive) timestamp in: query - schema: *ref_127 + schema: *ref_129 StartedBefore: name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_72 + schema: *ref_74 StartedAfter: name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_73 + schema: *ref_75 Before: name: before description: filter on started before (inclusive) timestamp in: query - schema: *ref_118 + schema: *ref_120 CreatedOrStartedAfter: name: created_or_started_after description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query - schema: *ref_121 + schema: *ref_123 CreatedOrStartedAfterCompletedJob: name: created_or_started_after_completed_jobs description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp but only for the completed jobs in: query - schema: *ref_122 + schema: *ref_124 CreatedOrStartedBefore: name: created_or_started_before description: >- filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query - schema: *ref_120 + schema: *ref_122 Success: name: success description: filter on successful jobs in: query - schema: *ref_74 + schema: *ref_76 ScheduledForBeforeNow: name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: *ref_75 + schema: *ref_77 Suspended: name: suspended description: filter on suspended jobs in: query - schema: *ref_77 + schema: *ref_79 Running: name: running description: filter on running jobs in: query - schema: *ref_78 + schema: *ref_80 ArgsFilter: name: args description: filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_79 + schema: *ref_81 Tag: name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_81 + schema: *ref_83 ResultFilter: name: result description: filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_80 + schema: *ref_82 After: name: after description: filter on created after (exclusive) timestamp in: query - schema: *ref_119 + schema: *ref_121 Username: name: username description: filter on exact username of user in: query - schema: *ref_128 + schema: *ref_130 Operation: name: operation description: filter on exact or prefix name of operation in: query - schema: *ref_129 + schema: *ref_131 ResourceName: name: resource description: filter on exact or prefix name of resource in: query - schema: *ref_130 + schema: *ref_132 ActionKind: name: action_kind description: filter on type of operation in: query - schema: *ref_131 + schema: *ref_133 JobKinds: name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_76 + schema: *ref_78 RunnableId: name: runnable_id in: query - schema: *ref_110 + schema: *ref_112 RunnableTypeQuery: name: runnable_type in: query - schema: *ref_111 + schema: *ref_113 InputId: name: input in: path required: true - schema: *ref_132 + schema: *ref_134 GetStarted: name: get_started in: query - schema: *ref_133 + schema: *ref_135 ConcurrencyId: name: concurrency_id in: path required: true - schema: *ref_134 + schema: *ref_136 schemas: + AiResource: + type: object + properties: *ref_16 + required: *ref_17 Script: type: object - properties: *ref_35 - required: *ref_36 + properties: *ref_37 + required: *ref_38 NewScript: - type: object - properties: *ref_40 - required: *ref_41 - NewScriptWithDraft: - allOf: *ref_135 - ScriptHistory: type: object properties: *ref_42 required: *ref_43 + NewScriptWithDraft: + allOf: *ref_137 + ScriptHistory: + type: object + properties: *ref_44 + required: *ref_45 ScriptArgs: type: object - additionalProperties: *ref_16 + additionalProperties: *ref_18 Input: type: object - properties: *ref_112 - required: *ref_113 + properties: *ref_114 + required: *ref_115 CreateInput: - type: object - properties: *ref_136 - required: *ref_137 - UpdateInput: type: object properties: *ref_138 required: *ref_139 + UpdateInput: + type: object + properties: *ref_140 + required: *ref_141 RunnableType: type: string - enum: *ref_140 + enum: *ref_142 QueuedJob: + type: object + properties: *ref_89 + required: *ref_90 + CompletedJob: type: object properties: *ref_87 required: *ref_88 - CompletedJob: - type: object - properties: *ref_85 - required: *ref_86 ObscuredJob: type: object - properties: *ref_141 + properties: *ref_143 Job: - oneOf: *ref_90 - discriminator: *ref_91 + oneOf: *ref_92 + discriminator: *ref_93 User: type: object properties: *ref_10 required: *ref_11 UserUsage: type: object - properties: *ref_142 + properties: *ref_144 Login: type: object - properties: *ref_143 - required: *ref_144 + properties: *ref_145 + required: *ref_146 EditWorkspaceUser: type: object - properties: *ref_145 + properties: *ref_147 TruncatedToken: type: object - properties: *ref_38 - required: *ref_39 + properties: *ref_40 + required: *ref_41 NewToken: type: object - properties: *ref_146 + properties: *ref_148 NewTokenImpersonate: - type: object - properties: *ref_147 - required: *ref_148 - ListableVariable: - type: object - properties: *ref_22 - required: *ref_23 - ContextualVariable: type: object properties: *ref_149 required: *ref_150 - CreateVariable: + ListableVariable: + type: object + properties: *ref_24 + required: *ref_25 + ContextualVariable: type: object properties: *ref_151 required: *ref_152 - EditVariable: + CreateVariable: type: object properties: *ref_153 + required: *ref_154 + EditVariable: + type: object + properties: *ref_155 AuditLog: type: object properties: *ref_1 @@ -14909,108 +15051,108 @@ components: - no_main_func - has_preprocessor Preview: - type: object - properties: *ref_154 - required: *ref_155 - WorkflowTask: type: object properties: *ref_156 required: *ref_157 + WorkflowTask: + type: object + properties: *ref_158 + required: *ref_159 WorkflowStatusRecord: type: object - additionalProperties: *ref_158 + additionalProperties: *ref_160 WorkflowStatus: type: object - properties: *ref_159 + properties: *ref_161 CreateResource: type: object - properties: *ref_160 - required: *ref_161 + properties: *ref_162 + required: *ref_163 EditResource: type: object - properties: *ref_162 + properties: *ref_164 Resource: - type: object - properties: *ref_163 - required: *ref_164 - ListableResource: type: object properties: *ref_165 required: *ref_166 - ResourceType: - type: object - properties: *ref_26 - required: *ref_27 - EditResourceType: + ListableResource: type: object properties: *ref_167 - Schedule: + required: *ref_168 + ResourceType: type: object - properties: *ref_94 - required: *ref_95 - ScheduleWJobs: - allOf: *ref_168 - NewSchedule: + properties: *ref_28 + required: *ref_29 + EditResourceType: type: object properties: *ref_169 - required: *ref_170 - EditSchedule: - type: object - properties: *ref_171 - required: *ref_172 - HttpTrigger: + Schedule: type: object properties: *ref_96 required: *ref_97 - NewHttpTrigger: + ScheduleWJobs: + allOf: *ref_170 + NewSchedule: + type: object + properties: *ref_171 + required: *ref_172 + EditSchedule: type: object properties: *ref_173 required: *ref_174 - EditHttpTrigger: + HttpTrigger: + type: object + properties: *ref_98 + required: *ref_99 + NewHttpTrigger: type: object properties: *ref_175 required: *ref_176 - TriggersCount: - type: object - properties: *ref_57 - WebsocketTrigger: - type: object - properties: *ref_99 - required: *ref_100 - NewWebsocketTrigger: + EditHttpTrigger: type: object properties: *ref_177 required: *ref_178 - EditWebsocketTrigger: + TriggersCount: + type: object + properties: *ref_59 + WebsocketTrigger: + type: object + properties: *ref_101 + required: *ref_102 + NewWebsocketTrigger: type: object properties: *ref_179 required: *ref_180 - WebsocketTriggerInitialMessage: - anyOf: *ref_98 - Group: - type: object - properties: *ref_106 - required: *ref_107 - InstanceGroup: - type: object - properties: *ref_102 - required: *ref_103 - Folder: - type: object - properties: *ref_108 - required: *ref_109 - WorkerPing: + EditWebsocketTrigger: type: object properties: *ref_181 required: *ref_182 - UserWorkspaceList: + WebsocketTriggerInitialMessage: + anyOf: *ref_100 + Group: + type: object + properties: *ref_108 + required: *ref_109 + InstanceGroup: + type: object + properties: *ref_104 + required: *ref_105 + Folder: + type: object + properties: *ref_110 + required: *ref_111 + WorkerPing: type: object properties: *ref_183 required: *ref_184 - CreateWorkspace: + UserWorkspaceList: type: object properties: *ref_185 required: *ref_186 + CreateWorkspace: + type: object + properties: *ref_187 + required: *ref_188 Workspace: type: object properties: *ref_7 @@ -15024,48 +15166,48 @@ components: properties: *ref_12 required: *ref_13 Flow: - allOf: *ref_56 + allOf: *ref_58 ExtraPerms: type: object - additionalProperties: *ref_187 + additionalProperties: *ref_189 FlowMetadata: - type: object - properties: *ref_188 - required: *ref_189 - OpenFlowWPath: - allOf: *ref_58 - FlowPreview: type: object properties: *ref_190 required: *ref_191 - RestartedFrom: + OpenFlowWPath: + allOf: *ref_60 + FlowPreview: type: object properties: *ref_192 + required: *ref_193 + RestartedFrom: + type: object + properties: *ref_194 Policy: type: object - properties: *ref_59 + properties: *ref_61 ListableApp: - type: object - properties: *ref_193 - required: *ref_194 - ListableRawApp: type: object properties: *ref_195 required: *ref_196 - AppWithLastVersion: + ListableRawApp: type: object - properties: *ref_60 - required: *ref_61 - AppWithLastVersionWDraft: - allOf: *ref_197 - AppHistory: + properties: *ref_197 + required: *ref_198 + AppWithLastVersion: type: object properties: *ref_62 required: *ref_63 + AppWithLastVersionWDraft: + allOf: *ref_199 + AppHistory: + type: object + properties: *ref_64 + required: *ref_65 FlowVersion: type: object - properties: *ref_54 - required: *ref_55 + properties: *ref_56 + required: *ref_57 SlackToken: type: object properties: @@ -15086,47 +15228,47 @@ components: - team_name - bot TokenResponse: - type: object - properties: *ref_198 - required: *ref_199 - HubScriptKind: - name: kind - schema: *ref_34 - PolarsClientKwargs: - type: object - properties: *ref_116 - required: *ref_117 - LargeFileStorage: - type: object - properties: *ref_17 - WindmillLargeFile: type: object properties: *ref_200 required: *ref_201 - WindmillFileMetadata: + HubScriptKind: + name: kind + schema: *ref_36 + PolarsClientKwargs: type: object - properties: *ref_202 - WindmillFilePreview: - type: object - properties: *ref_203 - required: *ref_204 - S3Resource: - type: object - properties: *ref_114 - required: *ref_115 - WorkspaceGitSyncSettings: - type: object - properties: *ref_18 - WorkspaceDeployUISettings: + properties: *ref_118 + required: *ref_119 + LargeFileStorage: type: object properties: *ref_19 - WorkspaceDefaultScripts: + WindmillLargeFile: type: object - properties: *ref_20 - GitRepositorySettings: + properties: *ref_202 + required: *ref_203 + WindmillFileMetadata: + type: object + properties: *ref_204 + WindmillFilePreview: type: object properties: *ref_205 required: *ref_206 + S3Resource: + type: object + properties: *ref_116 + required: *ref_117 + WorkspaceGitSyncSettings: + type: object + properties: *ref_20 + WorkspaceDeployUISettings: + type: object + properties: *ref_21 + WorkspaceDefaultScripts: + type: object + properties: *ref_22 + GitRepositorySettings: + type: object + properties: *ref_207 + required: *ref_208 UploadFilePart: type: object properties: @@ -15138,127 +15280,127 @@ components: - part_number - tag MetricMetadata: - type: object - properties: *ref_207 - required: *ref_208 - ScalarMetric: type: object properties: *ref_209 required: *ref_210 - TimeseriesMetric: + ScalarMetric: type: object properties: *ref_211 required: *ref_212 - MetricDataPoint: + TimeseriesMetric: type: object properties: *ref_213 required: *ref_214 - RawScriptForDependencies: + MetricDataPoint: type: object properties: *ref_215 required: *ref_216 - ConcurrencyGroup: + RawScriptForDependencies: type: object properties: *ref_217 required: *ref_218 - ExtendedJobs: + ConcurrencyGroup: type: object properties: *ref_219 required: *ref_220 + ExtendedJobs: + type: object + properties: *ref_221 + required: *ref_222 ExportedUser: type: object properties: *ref_3 required: *ref_4 GlobalSetting: - type: object - properties: *ref_221 - required: *ref_222 - Config: type: object properties: *ref_223 required: *ref_224 - ExportedInstanceGroup: - type: object - properties: *ref_104 - required: *ref_105 - JobSearchHit: + Config: type: object properties: *ref_225 - LogSearchHit: + required: *ref_226 + ExportedInstanceGroup: type: object - properties: *ref_226 - AutoscalingEvent: + properties: *ref_106 + required: *ref_107 + JobSearchHit: type: object properties: *ref_227 - CriticalAlert: + LogSearchHit: type: object properties: *ref_228 - StaticTransform: + AutoscalingEvent: type: object properties: *ref_229 - required: *ref_230 - JavascriptTransform: + CriticalAlert: + type: object + properties: *ref_230 + StaticTransform: type: object properties: *ref_231 required: *ref_232 - InputTransform: - oneOf: *ref_29 - discriminator: *ref_30 - RawScript: + JavascriptTransform: type: object properties: *ref_233 required: *ref_234 - PathScript: + InputTransform: + oneOf: *ref_31 + discriminator: *ref_32 + RawScript: type: object properties: *ref_235 required: *ref_236 - PathFlow: + PathScript: type: object properties: *ref_237 required: *ref_238 - FlowModule: - type: object - properties: *ref_31 - required: *ref_32 - ForloopFlow: + PathFlow: type: object properties: *ref_239 required: *ref_240 - WhileloopFlow: + FlowModule: + type: object + properties: *ref_33 + required: *ref_34 + ForloopFlow: type: object properties: *ref_241 required: *ref_242 - BranchOne: + WhileloopFlow: type: object properties: *ref_243 required: *ref_244 - BranchAll: + BranchOne: type: object properties: *ref_245 required: *ref_246 - Identity: + BranchAll: type: object properties: *ref_247 required: *ref_248 + Identity: + type: object + properties: *ref_249 + required: *ref_250 FlowModuleValue: - oneOf: *ref_249 - discriminator: *ref_250 + oneOf: *ref_251 + discriminator: *ref_252 Retry: type: object - properties: *ref_93 + properties: *ref_95 FlowValue: - type: object - properties: *ref_64 - required: *ref_65 - OpenFlow: - type: object - properties: *ref_52 - required: *ref_53 - FlowStatusModule: type: object properties: *ref_66 required: *ref_67 + OpenFlow: + type: object + properties: *ref_54 + required: *ref_55 + FlowStatusModule: + type: object + properties: *ref_68 + required: *ref_69 FlowStatus: type: object - properties: *ref_82 - required: *ref_83 + properties: *ref_84 + required: *ref_85 From 2a78359af7b8e4700634c2ad2745c1d14d2da4f7 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 14 Nov 2024 20:44:29 +0100 Subject: [PATCH 09/31] fix: add countCompletedJobs api --- backend/windmill-api/openapi.yaml | 33 +++++++++++++++++++++ backend/windmill-api/src/jobs.rs | 45 +++++++++++++++++++++++++++++ python-client/wmill/wmill/client.py | 2 +- 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ade446bf67..c1f755b53c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6054,6 +6054,39 @@ paths: required: - database_length + /w/{workspace}/jobs/completed/count_jobs: + get: + summary: count number of completed jobs with filter + operationId: countCompletedJobs + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: completed_after_s_ago + in: query + schema: + type: integer + - name: success + in: query + schema: + type: boolean + - name: tags + in: query + schema: + type: string + - name: all_workspaces + in: query + schema: + type: boolean + responses: + "200": + description: Count of completed jobs + content: + application/json: + schema: + type: integer + + /w/{workspace}/jobs/queue/list_filtered_uuids: get: summary: get the ids of all jobs matching the given filters diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 680bfd6c76..3c7cea1584 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -206,6 +206,7 @@ pub fn workspaced_service() -> Router { .route("/queue/list_filtered_uuids", get(list_filtered_uuids)) .route("/queue/cancel_selection", post(cancel_selection)) .route("/completed/count", get(count_completed_jobs)) + .route("/completed/count_jobs", get(count_completed_jobs_detail)) .route( "/completed/list", get(list_completed_jobs).layer(cors.clone()), @@ -1592,6 +1593,50 @@ async fn count_queue_jobs( )) } +#[derive(Deserialize)] +pub struct CountCompletedJobsQuery { + completed_after_s_ago: Option, + success: Option, + tags: Option, + all_workspaces: Option, +} + +async fn count_completed_jobs_detail( + Extension(db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> error::JsonResult { + let mut sqlb = SqlBuilder::select_from("completed_job"); + sqlb + .field("COUNT(*) as count"); + + if !query.all_workspaces.unwrap_or(false) { + sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); + } + + if let Some(after_s_ago) = query.completed_after_s_ago { + let after = Utc::now() - chrono::Duration::seconds(after_s_ago); + sqlb.and_where_gt("started_at + duration_ms / 1000 * interval '1 second'", "?".bind(&after.to_rfc3339())); + } + + if let Some(success) = query.success { + sqlb.and_where_eq("success", "?".bind(&success)); + } + + if let Some(tags) = query.tags { + sqlb.and_where_in("tag", &tags.split(",").map(|t| format!("'{}'", t)).collect::>()); + } + + let sql = sqlb.sql()?; + let stats = sqlx::query_scalar::<_, i64>(&sql) + .fetch_one(&db) + .await?; + + Ok(Json(stats)) +} + + + async fn count_completed_jobs( Extension(db): Extension, Path(w_id): Path, diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index 08338a273f..c57e181a0f 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -26,7 +26,7 @@ JobStatus = Literal["RUNNING", "WAITING", "COMPLETED"] class Windmill: def __init__(self, base_url=None, token=None, workspace=None, verify=True): - base = base_url or os.environ.get("BASE_INTERNAL_URL") + base = base_url or os.environ.get("BASE_INTERNAL_URL") or os.environ.get("WM_BASE_URL") self.base_url = f"{base}/api" self.token = token or os.environ.get("WM_TOKEN") From c44fc4f5ab8bcb5dffec08d5ed4ccd61feb1cb13 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 14 Nov 2024 21:03:26 +0100 Subject: [PATCH 10/31] fix: deployment callbacks have a concurrency limit of 1 on same path --- backend/ee-repo-ref.txt | 2 +- backend/windmill-queue/src/jobs.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index e784771cb7..55cf6b995c 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -15602d5972e79192ff8f121cd783613b8ab365fa \ No newline at end of file +177895136549cdb7c2f2a26ba3b654d76e60d845 \ No newline at end of file diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 275034efe7..dd91226c7e 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -3670,15 +3670,15 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( } JobPayload::DeploymentCallback { path } => ( None, - Some(path), + Some(path.clone()), None, JobKind::DeploymentCallback, None, None, None, - None, - None, - None, + Some(format!("{workspace_id}:git_sync")), + Some(1), + Some(0), None, None, None, From 47facb38260fca99bb68a132efa34ca3187bdfe5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 14 Nov 2024 21:28:20 +0100 Subject: [PATCH 11/31] chore(main): release 1.424.0 (#4709) * chore(main): release 1.424.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel --- CHANGELOG.md | 17 +++++ backend/Cargo.lock | 64 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 66 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dfc4d73b5..ac102364e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## [1.424.0](https://github.com/windmill-labs/windmill/compare/v1.423.2...v1.424.0) (2024-11-14) + + +### Features + +* allow setting password and login type from superadmin UI ([2a1bff3](https://github.com/windmill-labs/windmill/commit/2a1bff3160b65eadba399229b5f53950ec2c0d48)) +* **backend:** monitor minimal version of living workers ([#4704](https://github.com/windmill-labs/windmill/issues/4704)) ([50ff183](https://github.com/windmill-labs/windmill/commit/50ff183baeaa2a39c2b0188ee8dd6172b08ae801)) +* Support mistral anthropic for ai ([#4692](https://github.com/windmill-labs/windmill/issues/4692)) ([556b4a4](https://github.com/windmill-labs/windmill/commit/556b4a41a1d010bb8a1796f485d343c1e412d278)) + + +### Bug Fixes + +* add countCompletedJobs api ([2a78359](https://github.com/windmill-labs/windmill/commit/2a78359af7b8e4700634c2ad2745c1d14d2da4f7)) +* add queue_couts api ([10b6b1d](https://github.com/windmill-labs/windmill/commit/10b6b1dc04cb2b2d4ec5ceb26a5dbd2ac7bd1394)) +* autoscaling when count < min worker set to min_workers ([180bb82](https://github.com/windmill-labs/windmill/commit/180bb826436f9960def8c6cf3e9692714ffdf917)) +* deployment callbacks have a concurrency limit of 1 on same path ([c44fc4f](https://github.com/windmill-labs/windmill/commit/c44fc4f5ab8bcb5dffec08d5ed4ccd61feb1cb13)) + ## [1.423.2](https://github.com/windmill-labs/windmill/compare/v1.423.1...v1.423.2) (2024-11-13) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f239021c99..7d1ff0ad2d 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -843,9 +843,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53dcf5e7d9bd1517b8b998e170e650047cea8a2b85fe1835abe3210713e541b7" +checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1598,9 +1598,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aeb932158bd710538c73702db6945cb68a8fb08c519e6e12706b94263b36db8" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ "jobserver", "libc", @@ -1783,9 +1783,9 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.1.2" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" +checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9" dependencies = [ "strum 0.26.3", "strum_macros 0.26.4", @@ -3355,9 +3355,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", "libz-sys", @@ -10610,7 +10610,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "axum", @@ -10652,7 +10652,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "argon2", @@ -10737,7 +10737,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.423.2" +version = "1.424.0" dependencies = [ "base64 0.22.1", "chrono", @@ -10755,7 +10755,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.423.2" +version = "1.424.0" dependencies = [ "chrono", "serde", @@ -10768,7 +10768,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "rsmq_async", @@ -10783,7 +10783,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "async-stream", @@ -10830,7 +10830,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.423.2" +version = "1.424.0" dependencies = [ "regex", "rsmq_async", @@ -10845,7 +10845,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "bytes", @@ -10867,7 +10867,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.423.2" +version = "1.424.0" dependencies = [ "itertools 0.13.0", "lazy_static", @@ -10879,7 +10879,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.423.2" +version = "1.424.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -10888,7 +10888,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "lazy_static", @@ -10900,7 +10900,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "gosyn", @@ -10912,7 +10912,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "lazy_static", @@ -10924,7 +10924,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10935,7 +10935,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10946,7 +10946,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "async-recursion", @@ -10964,7 +10964,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10981,7 +10981,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "lazy_static", @@ -10993,7 +10993,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "lazy_static", @@ -11011,7 +11011,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -11032,7 +11032,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "serde_json", @@ -11042,7 +11042,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "async-recursion", @@ -11075,7 +11075,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.423.2" +version = "1.424.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -11085,7 +11085,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.423.2" +version = "1.424.0" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 51bf578c96..9b59a058d2 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.423.2" +version = "1.424.0" authors.workspace = true edition.workspace = true @@ -29,7 +29,7 @@ members = [ ] [workspace.package] -version = "1.423.2" +version = "1.424.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c1f755b53c..9d99cf960d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.423.2 + version: 1.424.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index d59c074be2..d8f1cd7bf4 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.423.2"; +export const VERSION = "v1.424.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index c35b0c7a27..b95ffa3314 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.423.2"; +export const VERSION = "1.424.0"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4b63b77cf2..58fef66ef0 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.423.2", + "version": "1.424.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.423.2", + "version": "1.424.0", "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.32.1", diff --git a/frontend/package.json b/frontend/package.json index 5b521f7d77..830b0411fc 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.423.2", + "version": "1.424.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 7bc5bf2c65..25c1bfaf01 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.423.2" -wmill_pg = ">=1.423.2" +wmill = ">=1.424.0" +wmill_pg = ">=1.424.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 535d9e22c4..ff3a2f2e4b 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.423.2 + version: 1.424.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 360a387370..b80d2870f0 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. -ModuleVersion = '1.423.2' +ModuleVersion = '1.424.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index efe6e0a58f..615bc73419 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.423.2" +version = "1.424.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 4f538fe6e3..498c47f302 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.423.2" +version = "1.424.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 47a093ea5b..ac3f5b871c 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.423.2", + "version": "1.424.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 10228562b8..5d03e77a44 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.423.2", + "version": "1.424.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 4141884b44..9e995e57ad 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.423.2 +1.424.0 From f240d1322a3c1669dfb69ebda6d05effcca26ae3 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Thu, 14 Nov 2024 23:12:56 +0000 Subject: [PATCH 12/31] feat: Handle `pip install` by `uv` (#4517) * feat: Handle `pip install` by `uv` Dirty and untested, but already something working * Integrate with NSJAIL and prepare fallbacks * Refactor fallback no_uv disable compile and install where no_uv_install and no_uv_compile are a bit more specific * Remove `--disable-pip-version-check` Reason: warning: pip's `--disable-pip-version-check` has no effect * Fix backend compilation error * Pip fallback overwrite UV's cache * Initially refactor cache (No S3) * Support S3 * Remove unused import * Handle flags for NSJAIL * Return deleted flag * Update Dockerfile * Update docker-image.yml * Update docker-image.yml * Add --link-mode=copy and remove -v * Fix NSJAIL INDEX_URL * Fix flock and windows * Update python_executor.rs * Remove line from Dockerfile We dont need it and to trigger build * fixing for windows * Dont pin python to specific version * Change TMP for windows * Revert docker-image.yml * Disable UV for ansible Will be enabled later. Needs proper testing and its better to split onto 2 PRs with first modifying python and second ansible --------- Co-authored-by: Ruben Fiszel Co-authored-by: Alexander Petric --- backend/src/main.rs | 5 +- backend/windmill-common/src/worker.rs | 2 + .../nsjail/download.py.pip.config.proto | 93 +++++++++++ .../nsjail/download_deps.py.pip.sh | 24 +++ .../nsjail/download_deps.py.sh | 14 +- .../windmill-worker/src/ansible_executor.rs | 2 + backend/windmill-worker/src/global_cache.rs | 26 +++- .../windmill-worker/src/python_executor.rs | 144 +++++++++++++----- backend/windmill-worker/src/worker.rs | 16 ++ .../windmill-worker/src/worker_lockfiles.rs | 2 + 10 files changed, 285 insertions(+), 43 deletions(-) create mode 100644 backend/windmill-worker/nsjail/download.py.pip.config.proto create mode 100755 backend/windmill-worker/nsjail/download_deps.py.pip.sh diff --git a/backend/src/main.rs b/backend/src/main.rs index 1f1fab319d..a711a2653e 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -65,8 +65,8 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING; use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, - GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, - RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, + GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, POWERSHELL_CACHE_DIR, RUST_CACHE_DIR, + TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -995,7 +995,6 @@ pub async fn run_workers(path: &str) -> Option { pub struct PythonAnnotations { pub no_cache: bool, pub no_uv: bool, + pub no_uv_install: bool, + pub no_uv_compile: bool, } #[annotations("//")] diff --git a/backend/windmill-worker/nsjail/download.py.pip.config.proto b/backend/windmill-worker/nsjail/download.py.pip.config.proto new file mode 100644 index 0000000000..15e5c6c7f6 --- /dev/null +++ b/backend/windmill-worker/nsjail/download.py.pip.config.proto @@ -0,0 +1,93 @@ +name: "python download pip" + +mode: ONCE +hostname: "python" +log_level: ERROR +time_limit: 900 + +rlimit_as: 2048 +rlimit_cpu: 1000 +rlimit_fsize: 1024 +rlimit_nofile: 64 + +envar: "HOME=/user" +envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +keep_caps: true +keep_env: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + dst: "/tmp" + fstype: "tmpfs" + rw: true + options: "size=500000000" +} + + +mount { + src: "{WORKER_DIR}/download_deps.py.pip.sh" + dst: "/download_deps.sh" + is_bind: true +} + +mount { + src: "{CACHE_DIR}" + dst: "{CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +exec_bin { + path: "/bin/sh" + arg: "/download_deps.sh" +} + diff --git a/backend/windmill-worker/nsjail/download_deps.py.pip.sh b/backend/windmill-worker/nsjail/download_deps.py.pip.sh new file mode 100755 index 0000000000..efd6274832 --- /dev/null +++ b/backend/windmill-worker/nsjail/download_deps.py.pip.sh @@ -0,0 +1,24 @@ +#/bin/sh + +INDEX_URL_ARG=$([ -z "$INDEX_URL" ] && echo ""|| echo "--index-url $INDEX_URL" ) +EXTRA_INDEX_URL_ARG=$([ -z "$EXTRA_INDEX_URL" ] && echo ""|| echo "--extra-index-url $EXTRA_INDEX_URL" ) +TRUSTED_HOST_ARG=$([ -z "$TRUSTED_HOST" ] && echo "" || echo "--trusted-host $TRUSTED_HOST") + +if [ ! -z "$INDEX_URL" ] +then + echo "\$INDEX_URL is set to $INDEX_URL" +fi + +if [ ! -z "$EXTRA_INDEX_URL" ] +then + echo "\$EXTRA_INDEX_URL is set to $EXTRA_INDEX_URL" +fi + +if [ ! -z "$TRUSTED_HOST" ] +then + echo "\$TRUSTED_HOST is set to $TRUSTED_HOST" +fi + +CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG" +echo $CMD +eval $CMD diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index efd6274832..3898282d2a 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -19,6 +19,18 @@ then echo "\$TRUSTED_HOST is set to $TRUSTED_HOST" fi -CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG" +CMD="/usr/local/bin/uv pip install +\"$REQ\" +--target \"$TARGET\" +--no-cache +--no-config +--no-color +--no-deps +--link-mode=copy +$INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG +--index-strategy unsafe-best-match +--system +" + echo $CMD eval $CMD diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index abd9b68415..3548219ca5 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -116,6 +116,8 @@ async fn handle_ansible_python_deps( job_dir, worker_dir, &mut Some(occupancy_metrics), + true, + true, ) .await?; additional_python_paths.append(&mut venv_path); diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 12584b905d..57b2b55885 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -20,13 +20,21 @@ use std::sync::Arc; pub async fn build_tar_and_push( s3_client: Arc, folder: String, + no_uv: bool, ) -> error::Result<()> { use object_store::path::Path; + use crate::PY311_CACHE_DIR; + tracing::info!("Started building and pushing piptar {folder}"); let start = Instant::now(); let folder_name = folder.split("/").last().unwrap(); - let tar_path = format!("{PIP_CACHE_DIR}/{folder_name}_tar.tar",); + let prefix = if no_uv { + PIP_CACHE_DIR + } else { + PY311_CACHE_DIR + }; + let tar_path = format!("{prefix}/{folder_name}_tar.tar",); let tar_file = std::fs::File::create(&tar_path)?; let mut tar = tar::Builder::new(tar_file); @@ -46,7 +54,10 @@ pub async fn build_tar_and_push( // })?; if let Err(e) = s3_client .put( - &Path::from(format!("/tar/pip/{folder_name}.tar")), + &Path::from(format!( + "/tar/{}/{folder_name}.tar", + if no_uv { "pip" } else { "python_311" } + )), std::fs::read(&tar_path)?.into(), ) .await @@ -71,7 +82,11 @@ pub async fn build_tar_and_push( } #[cfg(all(feature = "enterprise", feature = "parquet"))] -pub async fn pull_from_tar(client: Arc, folder: String) -> error::Result<()> { +pub async fn pull_from_tar( + client: Arc, + folder: String, + no_uv: bool, +) -> error::Result<()> { use windmill_common::s3_helpers::attempt_fetch_bytes; let folder_name = folder.split("/").last().unwrap(); @@ -79,7 +94,10 @@ pub async fn pull_from_tar(client: Arc, folder: String) -> erro tracing::info!("Attempting to pull piptar {folder_name} from bucket"); let start = Instant::now(); - let tar_path = format!("tar/pip/{folder_name}.tar"); + let tar_path = format!( + "tar/{}/{folder_name}.tar", + if no_uv { "pip" } else { "python_311" } + ); let bytes = attempt_fetch_bytes(client, &tar_path).await?; // tracing::info!("B: {target} {folder}"); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 9db4ed529b..b06f30b98d 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -42,6 +42,10 @@ lazy_static::lazy_static! { static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") .ok().map(|flag| flag == "true").unwrap_or(false); + /// Use pip install + static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") + .ok().map(|flag| flag == "true").unwrap_or(false); + static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); @@ -50,6 +54,8 @@ lazy_static::lazy_static! { } const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto"); +const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK: &str = + include_str!("../nsjail/download.py.pip.config.proto"); const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto"); const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); @@ -66,8 +72,8 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR, - NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, TZ_ENV, - UV_CACHE_DIR, + NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, + PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR, }; #[cfg(windows)] @@ -901,10 +907,10 @@ async fn handle_python_deps( .unwrap_or_else(|| vec![]) .clone(); + let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); let requirements = match requirements_o { Some(r) => r, None => { - let annotation = windmill_common::worker::PythonAnnotations::parse(inner_content); let mut already_visited = vec![]; let requirements = windmill_parser_py_imports::parse_python_imports( @@ -929,8 +935,8 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, - annotation.no_uv, - annotation.no_cache, + annotations.no_uv || annotations.no_uv_compile, + annotations.no_cache, ) .await .map_err(|e| { @@ -955,6 +961,8 @@ async fn handle_python_deps( job_dir, worker_dir, occupancy_metrics, + annotations.no_uv || annotations.no_uv_install, + false, ) .await?; additional_python_paths.append(&mut venv_path); @@ -966,6 +974,7 @@ lazy_static::lazy_static! { static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^\s\}]+)\}").unwrap(); } +/// pip install, include cached or pull from S3 pub async fn handle_python_reqs( requirements: Vec<&str>, job_id: &Uuid, @@ -977,12 +986,22 @@ pub async fn handle_python_reqs( job_dir: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + // TODO: Remove (Deprecated) + mut no_uv_install: bool, + is_ansible: bool, ) -> error::Result> { let mut req_paths: Vec = vec![]; let mut vars = vec![("PATH", PATH_ENV.as_str())]; let pip_extra_index_url; let pip_index_url; + no_uv_install |= *USE_PIP_INSTALL; + + if no_uv_install && !is_ansible { + append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await; + tracing::warn!("Fallback to pip"); + } + if !*DISABLE_NSJAIL { pip_extra_index_url = PIP_EXTRA_INDEX_URL .read() @@ -1013,10 +1032,21 @@ pub async fn handle_python_reqs( let _ = write_file( job_dir, "download.config.proto", - &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT - .replace("{WORKER_DIR}", &worker_dir) - .replace("{CACHE_DIR}", PIP_CACHE_DIR) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), + &(if no_uv_install { + NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK + } else { + NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT + }) + .replace("{WORKER_DIR}", &worker_dir) + .replace( + "{CACHE_DIR}", + if no_uv_install { + PIP_CACHE_DIR + } else { + PY311_CACHE_DIR + }, + ) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; }; @@ -1026,8 +1056,14 @@ pub async fn handle_python_reqs( if req.starts_with('#') { continue; } + let py_prefix = if no_uv_install { + PIP_CACHE_DIR + } else { + PY311_CACHE_DIR + }; + let venv_p = format!( - "{PIP_CACHE_DIR}/{}", + "{py_prefix}/{}", req.replace(' ', "").replace('/', "").replace(':', "") ); if metadata(&venv_p).await.is_ok() { @@ -1073,7 +1109,10 @@ pub async fn handle_python_reqs( .map(|(req, venv_p)| { let os = os.clone(); async move { - if pull_from_tar(os, venv_p.clone()).await.is_ok() { + if pull_from_tar(os, venv_p.clone(), no_uv_install) + .await + .is_ok() + { PullFromTar::Pulled(venv_p.to_string()) } else { PullFromTar::NotPulled(req.to_string(), venv_p.to_string()) @@ -1114,7 +1153,7 @@ pub async fn handle_python_reqs( for (req, venv_p) in req_with_penv { let mut logs1 = String::new(); - logs1.push_str("\n\n--- PIP INSTALL ---\n"); + logs1.push_str("\n\n--- UV PIP INSTALL ---\n"); logs1.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses.")); append_logs(&job_id, w_id, logs1, db).await; @@ -1150,21 +1189,47 @@ pub async fn handle_python_reqs( #[cfg(windows)] let req = format!("{}", req); - let mut command_args = vec![ - PYTHON_PATH.as_str(), - "-m", - "pip", - "install", - &req, - "-I", - "--no-deps", - "--no-color", - "--isolated", - "--no-warn-conflicts", - "--disable-pip-version-check", - "-t", - venv_p.as_str(), - ]; + let mut command_args = if no_uv_install { + vec![ + PYTHON_PATH.as_str(), + "-m", + "pip", + "install", + &req, + "-I", + "--no-deps", + "--no-color", + "--isolated", + "--no-warn-conflicts", + "--disable-pip-version-check", + "-t", + venv_p.as_str(), + ] + } else { + vec![ + UV_PATH.as_str(), + "pip", + "install", + &req, + "--no-deps", + "--no-color", + // "-p", + // "3.11", + // Prevent uv from discovering configuration files. + "--no-config", + "--link-mode=copy", + // TODO: Doublecheck it + "--system", + // Prefer main index over extra + // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes + // TODO: Use env variable that can be toggled from UI + "--index-strategy", + "unsafe-best-match", + "--target", + venv_p.as_str(), + "--no-cache", + ] + }; let pip_extra_index_url = PIP_EXTRA_INDEX_URL .read() .await @@ -1196,7 +1261,7 @@ pub async fn handle_python_reqs( envs.push(("HOME", HOME_ENV.as_str())); - tracing::debug!("pip install command: {:?}", command_args); + tracing::debug!("uv pip install command: {:?}", command_args); #[cfg(unix)] { @@ -1207,7 +1272,12 @@ pub async fn handle_python_reqs( .envs(envs) .args([ "-x", - &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), + &format!( + "{}/{}-{}.lock", + LOCK_CACHE_DIR, + if no_uv_install { "pip" } else { "py311" }, + fssafe_req + ), "--command", &command_args.join(" "), ]) @@ -1218,16 +1288,20 @@ pub async fn handle_python_reqs( #[cfg(windows)] { - let mut pip_cmd = Command::new(PYTHON_PATH.as_str()); - pip_cmd - .env_clear() + let installer_path = if no_uv_install { command_args[0] } else { "uv" }; + let mut cmd: Command = Command::new(&installer_path); + cmd.env_clear() .envs(envs) .envs(PROXY_ENVS.clone()) .env("SystemRoot", SYSTEM_ROOT.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) .args(&command_args[1..]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - start_child_process(pip_cmd, PYTHON_PATH.as_str()).await? + start_child_process(cmd, installer_path).await? } }; @@ -1240,7 +1314,7 @@ pub async fn handle_python_reqs( false, worker_name, &w_id, - &format!("pip install {req}"), + &format!("uv pip install {req}"), None, false, occupancy_metrics, @@ -1260,7 +1334,7 @@ pub async fn handle_python_reqs( tracing::warn!("S3 cache not available in the pro plan"); } else { let venv_p = venv_p.clone(); - tokio::spawn(build_tar_and_push(os, venv_p)); + tokio::spawn(build_tar_and_push(os, venv_p, no_uv_install)); } } req_paths.push(venv_p); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 28a94d5f92..e6e92e1682 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -236,7 +236,14 @@ pub const TMP_LOGS_DIR: &str = concatcp!(TMP_DIR, "/logs"); pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); +// Used as fallback now pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); + +// pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); +pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); +// pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); +// pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); + pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); @@ -257,6 +264,7 @@ const NUM_SECS_PING: u64 = 5; const NUM_SECS_READINGS: u64 = 60; const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); +const INCLUDE_DEPS_PY_SH_CONTENT_FALLBACK: &str = include_str!("../nsjail/download_deps.py.pip.sh"); pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900; pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days @@ -311,6 +319,7 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); + // pub static ref DISABLE_NSJAIL: bool = false; pub static ref DISABLE_NSJAIL: bool = std::env::var("DISABLE_NSJAIL") .ok() .and_then(|x| x.parse::().ok()) @@ -740,6 +749,13 @@ pub async fn run_worker Date: Fri, 15 Nov 2024 01:11:17 +0100 Subject: [PATCH 13/31] delete venv folder if pip install didn't succeed --- backend/windmill-worker/src/python_executor.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index b06f30b98d..c2b5dce572 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, process::Stdio}; +use std::{collections::HashMap, fs, process::Stdio}; use itertools::Itertools; use regex::Regex; @@ -1326,6 +1326,16 @@ pub async fn handle_python_reqs( "finished setting up python dependencies {}", job_id ); + if child.is_err() { + + if let Err(e) = fs::remove_dir_all(&venv_p) { + tracing::warn!( + workspace_id = %w_id, + "failed to remove cache dir: {:?}", + e + ); + } + } child?; #[cfg(all(feature = "enterprise", feature = "parquet"))] From 029462bc57fef0117fab7355b0c8edbe4fa965a9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 15 Nov 2024 01:15:51 +0100 Subject: [PATCH 14/31] nits logs --- backend/windmill-worker/src/python_executor.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index c2b5dce572..97ae820afc 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1153,7 +1153,11 @@ pub async fn handle_python_reqs( for (req, venv_p) in req_with_penv { let mut logs1 = String::new(); - logs1.push_str("\n\n--- UV PIP INSTALL ---\n"); + if no_uv_install { + logs1.push_str("\n\n--- PIP INSTALL ---\n"); + } else { + logs1.push_str("\n\n--- UV PIP INSTALL ---\n"); + } logs1.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses.")); append_logs(&job_id, w_id, logs1, db).await; From bb937498bbd0069a8757fd4ea5109eb9d19219d3 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Fri, 15 Nov 2024 02:03:05 +0000 Subject: [PATCH 15/31] Fix dirs in uv install (#4717) --- backend/src/main.rs | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index a711a2653e..d94537110a 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -30,14 +30,15 @@ use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICE use windmill_common::{ global_settings::{ - BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, - CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, - EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, - JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, - OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, - SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING, + BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, + EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, + LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, }, scripts::ScriptLang, stats_ee::schedule_stats, @@ -65,19 +66,20 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING; use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, - GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, POWERSHELL_CACHE_DIR, RUST_CACHE_DIR, - TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, + GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, + PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_require_preexisting_user, load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db, monitor_pool, reload_base_url_setting, reload_bunfig_install_scopes_setting, - reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, - reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_jwt_secret_setting, - reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting, + reload_critical_alert_mute_ui_setting, reload_critical_error_channels_setting, + reload_extra_pip_index_url_setting, reload_hub_base_url_setting, + reload_job_default_timeout_setting, reload_jwt_secret_setting, reload_license_key, + reload_npm_config_registry_setting, reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config, - reload_worker_config, reload_critical_alert_mute_ui_setting, + reload_worker_config, }; #[cfg(feature = "parquet")] @@ -1001,6 +1003,8 @@ pub async fn run_workers Date: Fri, 15 Nov 2024 09:36:18 +0100 Subject: [PATCH 16/31] improve error messages on windows --- backend/windmill-worker/src/common.rs | 12 +++++++++--- backend/windmill-worker/src/python_executor.rs | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index b406226120..31d8ecc4dd 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -41,7 +41,7 @@ use tokio::{io::AsyncWriteExt, process::Child, time::Instant}; use crate::{ AuthedClient, AuthedClientBackgroundTask, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, - MAX_TIMEOUT_DURATION, + MAX_TIMEOUT_DURATION, PATH_ENV, }; pub async fn build_args_map<'a>( @@ -860,11 +860,17 @@ pub async fn save_in_cache( } fn tentatively_improve_error(err: Error, executable: &str) -> Error { + #[cfg(unix)] + let err_msg = "No such file or directory (os error 2)"; + + #[cfg(windows)] + let err_msg = "program not found"; + if err .to_string() - .contains("No such file or directory (os error 2)") + .contains(&err_msg) { - return Error::InternalErr(format!("Executable {executable} not found on worker")); + return Error::InternalErr(format!("Executable {executable} not found on worker. PATH: {}", *PATH_ENV)); } return err; } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 97ae820afc..17feeff648 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -320,7 +320,7 @@ pub async fn uv_pip_compile( .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let child_process = start_child_process(child_cmd, "/usr/local/bin/uv").await?; + let child_process = start_child_process(child_cmd, uv_cmd).await?; append_logs(&job_id, &w_id, logs, db).await; handle_child( job_id, From c32038a76d9d00703d8b865af72f083ac43414e3 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 15 Nov 2024 03:48:48 -0500 Subject: [PATCH 17/31] feat(monitoring): workspace critical alerts (#4684) * critical alert ui * updating ui, backend logic * revert * type check fix npm * checking out cli files from main * moving alert icon * adding sqlx mock data * more sqlx changes * feat(frontend): nodes from flow can be connected directly in expr input through a plug icon (#4652) * Add flow prop picker # Conflicts: # frontend/src/lib/components/propertyPicker/PropPicker.svelte * fix unwanted copy * cleaning * Fix unset context * move button and always display input * fix unwanted proppicker display * update * update * clean all * clean all --------- Co-authored-by: Ruben Fiszel * replace hide/show with toggle * adding mutable setting and navigation to settings to configure channels * merge fix * ee non ee changees * auto-acknowledge when muted * pr comments * fix bad log * user inner modal component * update unaknowledge alerts after acknowledging from modal * aknowledge -> acknowledge * format * adding backend support for workspace critical alerts * immediately check for alerts * immediately check for alerts * adding openapi changes * simplify loading of superadmin/ee * update modal logic * frontend logic update * sqlx prepare * adding mute functionality for workspace critical alerts + show alerts button for instance and workspace settings * npm / rust warnings & errors * reverting non-mac cargo toml * reverting non-mac cargo toml * mute toggles in critical alert ui * ui polish * adding acknowledged_workspace column * sqlx prepare * make sure we wait for all stores to be loaded * adding workspace mute logic to report_critical alert * auto ack recovered alerts * toggle workspace as superadmin and critical alerts in logs menu * adding critical alert button if width < 786px * refresh on change * Move notification to logs (#4698) * Move notification to logs * removing unused button --------- Co-authored-by: Alexander Petric * tailwind typo * auto ack on mute * don't change deref * sqlx prep * avoid renaming db column, keep acknowledged instead of acknowledged_global * hide critical alert menu when not ee * if workspace muted, also acknowledge global when workspace set * removing save button for mute setting and improve ux/responsiveness * fix: deployment callbacks have a concurrency limit of 1 on same path * sqlx prep * ee-repo ref * z-[9999] for modal --------- Co-authored-by: Guilhem Co-authored-by: Ruben Fiszel Co-authored-by: Ruben Fiszel --- ...acb0f4c244c60477427505bf5bd1b104d3bf9.json | 14 ++ ...35e239cf7567d18b8b3fa6608ea3a9d206ca7.json | 18 ++ ...f29876724922f148dad7af35b36efbf70207a.json | 14 -- ...c61296a3ff7489ae12f52a19f9543173ac597.json | 6 + ...c2da733e71f8a253b0122c17b9e021919809e.json | 18 ++ ...5ab9fdff1a7252198e4ae290d8c8f937ff177.json | 54 +++++ ...48cd9ff63e5fdfa1d93d8fbec32a257b6b05e.json | 15 -- ...ec72884f2a1eacf0ab0e0c6ccbf56c654c14a.json | 14 ++ ...fa1848ed3c1336fa4790f45b8189c4ac97d91.json | 15 -- ...5816583f0bae78a0dd3270dedb377dfa944a.json} | 11 +- ...dcc40f463cbc52d94ed9315cf9a547d4c89f2.json | 6 + ...c068b72294cc99ff44ed0a13179df508ebc6a.json | 15 ++ ...43c8ead47a6a531d4b031a1342c9b2f16506.json} | 11 +- ...36caae3fa256fc0803749ec5107632669adb3.json | 14 ++ ...9714c20b767508e27368be6de38d7b18a3a4d.json | 55 +++++ ...3d2b652618ac454f139db738f4499cb4eb079.json | 12 ++ ...46976ed7af2e90270902f001115e023cb947d.json | 12 -- ...25f862dcf039bbb3d75ced9d54c86f53d2580.json | 14 -- ...872c721169d418586be2d15cee6b929049098.json | 22 ++ ...1e906a7868f34c062f154b2ee4f24c57aa5c3.json | 15 ++ backend/ee-repo-ref.txt | 2 +- ...1108201445_alert_add_workspace_id.down.sql | 4 + ...241108201445_alert_add_workspace_id.up.sql | 4 + backend/src/monitor.rs | 41 ++-- backend/windmill-api/openapi.yaml | 112 ++++++++++ backend/windmill-api/src/settings.rs | 77 +------ backend/windmill-api/src/utils.rs | 162 ++++++++++++++ .../windmill-api/src/websocket_triggers.rs | 8 +- backend/windmill-api/src/workspaces.rs | 97 ++++++++- backend/windmill-common/src/utils.rs | 120 +++++++++-- backend/windmill-queue/src/jobs.rs | 8 +- .../lib/components/InstanceSettings.svelte | 23 +- .../lib/components/SuperadminSettings.svelte | 2 +- .../lib/components/common/modal/Modal.svelte | 2 +- .../src/lib/components/instanceSettings.ts | 2 +- .../sidebar/CriticalAlertButton.svelte | 64 ------ .../sidebar/CriticalAlertModal.svelte | 73 ++++++- .../sidebar/CriticalAlertModalInner.svelte | 201 ++++++++++++++---- .../lib/components/sidebar/MenuButton.svelte | 15 +- .../sidebar/SideBarNotification.svelte | 14 ++ .../components/sidebar/SidebarContent.svelte | 81 +++++-- frontend/src/lib/stores.ts | 4 +- .../src/routes/(root)/(logged)/+layout.svelte | 74 ++++--- .../(logged)/workspace_settings/+page.svelte | 66 +++++- 44 files changed, 1250 insertions(+), 361 deletions(-) create mode 100644 backend/.sqlx/query-00588a40dde5189ac1c61505f17acb0f4c244c60477427505bf5bd1b104d3bf9.json create mode 100644 backend/.sqlx/query-044e2b428ee6e2dd4543c87ad8835e239cf7567d18b8b3fa6608ea3a9d206ca7.json delete mode 100644 backend/.sqlx/query-0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a.json create mode 100644 backend/.sqlx/query-28987898c7e4b172d466bf08f33c2da733e71f8a253b0122c17b9e021919809e.json create mode 100644 backend/.sqlx/query-344b3a5d9683273a956b5156fed5ab9fdff1a7252198e4ae290d8c8f937ff177.json delete mode 100644 backend/.sqlx/query-3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e.json create mode 100644 backend/.sqlx/query-4a3a8207627418ba7b7eabaccd7ec72884f2a1eacf0ab0e0c6ccbf56c654c14a.json delete mode 100644 backend/.sqlx/query-4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91.json rename backend/.sqlx/{query-0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995.json => query-4d30c5a2894d655741d167f5589f5816583f0bae78a0dd3270dedb377dfa944a.json} (63%) create mode 100644 backend/.sqlx/query-65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a.json rename backend/.sqlx/{query-cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b.json => query-777e7084edf9a5d14f299ff479ae43c8ead47a6a531d4b031a1342c9b2f16506.json} (60%) create mode 100644 backend/.sqlx/query-7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3.json create mode 100644 backend/.sqlx/query-7c5a29de07cbe42326a15d4d7fd9714c20b767508e27368be6de38d7b18a3a4d.json create mode 100644 backend/.sqlx/query-a59fae29ebcc9aa53308b777ead3d2b652618ac454f139db738f4499cb4eb079.json delete mode 100644 backend/.sqlx/query-a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d.json delete mode 100644 backend/.sqlx/query-be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580.json create mode 100644 backend/.sqlx/query-e05dbe046e846c092a96b6b0a9d872c721169d418586be2d15cee6b929049098.json create mode 100644 backend/.sqlx/query-f22168826350797e88153b65a5b1e906a7868f34c062f154b2ee4f24c57aa5c3.json create mode 100644 backend/migrations/20241108201445_alert_add_workspace_id.down.sql create mode 100644 backend/migrations/20241108201445_alert_add_workspace_id.up.sql delete mode 100644 frontend/src/lib/components/sidebar/CriticalAlertButton.svelte create mode 100644 frontend/src/lib/components/sidebar/SideBarNotification.svelte diff --git a/backend/.sqlx/query-00588a40dde5189ac1c61505f17acb0f4c244c60477427505bf5bd1b104d3bf9.json b/backend/.sqlx/query-00588a40dde5189ac1c61505f17acb0f4c244c60477427505bf5bd1b104d3bf9.json new file mode 100644 index 0000000000..c54a4c6bea --- /dev/null +++ b/backend/.sqlx/query-00588a40dde5189ac1c61505f17acb0f4c244c60477427505bf5bd1b104d3bf9.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE alerts SET acknowledged_workspace = true, acknowledged = true WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "00588a40dde5189ac1c61505f17acb0f4c244c60477427505bf5bd1b104d3bf9" +} diff --git a/backend/.sqlx/query-044e2b428ee6e2dd4543c87ad8835e239cf7567d18b8b3fa6608ea3a9d206ca7.json b/backend/.sqlx/query-044e2b428ee6e2dd4543c87ad8835e239cf7567d18b8b3fa6608ea3a9d206ca7.json new file mode 100644 index 0000000000..32d381e921 --- /dev/null +++ b/backend/.sqlx/query-044e2b428ee6e2dd4543c87ad8835e239cf7567d18b8b3fa6608ea3a9d206ca7.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource)\n VALUES ('critical_error', $1, $2, $3, $4, $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Bool", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "044e2b428ee6e2dd4543c87ad8835e239cf7567d18b8b3fa6608ea3a9d206ca7" +} diff --git a/backend/.sqlx/query-0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a.json b/backend/.sqlx/query-0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a.json deleted file mode 100644 index 5cb52b6ead..0000000000 --- a/backend/.sqlx/query-0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM healthchecks WHERE check_type = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [] - }, - "hash": "0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a" -} diff --git a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json index 6189a1fdab..686158d9a1 100644 --- a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json +++ b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json @@ -117,6 +117,11 @@ "ordinal": 22, "name": "deploy_ui", "type_info": "Jsonb" + }, + { + "ordinal": 23, + "name": "mute_critical_alerts", + "type_info": "Bool" } ], "parameters": { @@ -147,6 +152,7 @@ true, false, true, + true, true ] }, diff --git a/backend/.sqlx/query-28987898c7e4b172d466bf08f33c2da733e71f8a253b0122c17b9e021919809e.json b/backend/.sqlx/query-28987898c7e4b172d466bf08f33c2da733e71f8a253b0122c17b9e021919809e.json new file mode 100644 index 0000000000..63a46f634c --- /dev/null +++ b/backend/.sqlx/query-28987898c7e4b172d466bf08f33c2da733e71f8a253b0122c17b9e021919809e.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource)\n VALUES ('recovered_critical_error', $1, $2, $3, $4, $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Bool", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "28987898c7e4b172d466bf08f33c2da733e71f8a253b0122c17b9e021919809e" +} diff --git a/backend/.sqlx/query-344b3a5d9683273a956b5156fed5ab9fdff1a7252198e4ae290d8c8f937ff177.json b/backend/.sqlx/query-344b3a5d9683273a956b5156fed5ab9fdff1a7252198e4ae290d8c8f937ff177.json new file mode 100644 index 0000000000..7b47efe873 --- /dev/null +++ b/backend/.sqlx/query-344b3a5d9683273a956b5156fed5ab9fdff1a7252198e4ae290d8c8f937ff177.json @@ -0,0 +1,54 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id\n FROM alerts\n WHERE COALESCE(acknowledged, false) = $1\n ORDER BY created_at DESC\n LIMIT $2 OFFSET $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "alert_type", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "message", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "acknowledged", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "workspace_id", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Bool", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + null, + true + ] + }, + "hash": "344b3a5d9683273a956b5156fed5ab9fdff1a7252198e4ae290d8c8f937ff177" +} diff --git a/backend/.sqlx/query-3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e.json b/backend/.sqlx/query-3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e.json deleted file mode 100644 index 5a79a3d793..0000000000 --- a/backend/.sqlx/query-3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('recovered_critical_error', $1, $2)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e" -} diff --git a/backend/.sqlx/query-4a3a8207627418ba7b7eabaccd7ec72884f2a1eacf0ab0e0c6ccbf56c654c14a.json b/backend/.sqlx/query-4a3a8207627418ba7b7eabaccd7ec72884f2a1eacf0ab0e0c6ccbf56c654c14a.json new file mode 100644 index 0000000000..4d3b8ab93a --- /dev/null +++ b/backend/.sqlx/query-4a3a8207627418ba7b7eabaccd7ec72884f2a1eacf0ab0e0c6ccbf56c654c14a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE alerts SET acknowledged = true, acknowledged_workspace = true WHERE resource = $1 AND alert_type = 'critical_error'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4a3a8207627418ba7b7eabaccd7ec72884f2a1eacf0ab0e0c6ccbf56c654c14a" +} diff --git a/backend/.sqlx/query-4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91.json b/backend/.sqlx/query-4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91.json deleted file mode 100644 index b57d834f87..0000000000 --- a/backend/.sqlx/query-4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('critical_error', $1, $2)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91" -} diff --git a/backend/.sqlx/query-0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995.json b/backend/.sqlx/query-4d30c5a2894d655741d167f5589f5816583f0bae78a0dd3270dedb377dfa944a.json similarity index 63% rename from backend/.sqlx/query-0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995.json rename to backend/.sqlx/query-4d30c5a2894d655741d167f5589f5816583f0bae78a0dd3270dedb377dfa944a.json index 820e4f45ab..a8c371718b 100644 --- a/backend/.sqlx/query-0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995.json +++ b/backend/.sqlx/query-4d30c5a2894d655741d167f5589f5816583f0bae78a0dd3270dedb377dfa944a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, alert_type, message, created_at, acknowledged \n FROM alerts \n WHERE acknowledged = $1\n ORDER BY created_at DESC \n LIMIT $2 OFFSET $3", + "query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id\n FROM alerts\n ORDER BY created_at DESC\n LIMIT $1 OFFSET $2", "describe": { "columns": [ { @@ -27,11 +27,15 @@ "ordinal": 4, "name": "acknowledged", "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "workspace_id", + "type_info": "Text" } ], "parameters": { "Left": [ - "Bool", "Int8", "Int8" ] @@ -41,8 +45,9 @@ false, false, false, + null, true ] }, - "hash": "0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995" + "hash": "4d30c5a2894d655741d167f5589f5816583f0bae78a0dd3270dedb377dfa944a" } diff --git a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json b/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json index cfef50c160..6aab7e7796 100644 --- a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json +++ b/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json @@ -117,6 +117,11 @@ "ordinal": 22, "name": "deploy_ui", "type_info": "Jsonb" + }, + { + "ordinal": 23, + "name": "mute_critical_alerts", + "type_info": "Bool" } ], "parameters": { @@ -147,6 +152,7 @@ true, false, true, + true, true ] }, diff --git a/backend/.sqlx/query-65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a.json b/backend/.sqlx/query-65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a.json new file mode 100644 index 0000000000..e183d3b95c --- /dev/null +++ b/backend/.sqlx/query-65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE alerts\n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true\n ELSE acknowledged_workspace\n END\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Text" + ] + }, + "nullable": [] + }, + "hash": "65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a" +} diff --git a/backend/.sqlx/query-cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b.json b/backend/.sqlx/query-777e7084edf9a5d14f299ff479ae43c8ead47a6a531d4b031a1342c9b2f16506.json similarity index 60% rename from backend/.sqlx/query-cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b.json rename to backend/.sqlx/query-777e7084edf9a5d14f299ff479ae43c8ead47a6a531d4b031a1342c9b2f16506.json index 18bc159e6c..f25bba999a 100644 --- a/backend/.sqlx/query-cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b.json +++ b/backend/.sqlx/query-777e7084edf9a5d14f299ff479ae43c8ead47a6a531d4b031a1342c9b2f16506.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, alert_type, message, created_at, acknowledged \n FROM alerts \n ORDER BY created_at DESC \n LIMIT $1 OFFSET $2", + "query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id\n FROM alerts\n WHERE workspace_id = $1\n ORDER BY created_at DESC\n LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -27,10 +27,16 @@ "ordinal": 4, "name": "acknowledged", "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "workspace_id", + "type_info": "Text" } ], "parameters": { "Left": [ + "Text", "Int8", "Int8" ] @@ -40,8 +46,9 @@ false, false, false, + null, true ] }, - "hash": "cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b" + "hash": "777e7084edf9a5d14f299ff479ae43c8ead47a6a531d4b031a1342c9b2f16506" } diff --git a/backend/.sqlx/query-7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3.json b/backend/.sqlx/query-7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3.json new file mode 100644 index 0000000000..cf976de33f --- /dev/null +++ b/backend/.sqlx/query-7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE alerts \n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $1::text IS NOT NULL THEN true\n ELSE acknowledged_workspace\n END\n WHERE ($1::text IS NOT NULL AND workspace_id = $1)\n OR ($1::text IS NULL)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3" +} diff --git a/backend/.sqlx/query-7c5a29de07cbe42326a15d4d7fd9714c20b767508e27368be6de38d7b18a3a4d.json b/backend/.sqlx/query-7c5a29de07cbe42326a15d4d7fd9714c20b767508e27368be6de38d7b18a3a4d.json new file mode 100644 index 0000000000..7145fb2df9 --- /dev/null +++ b/backend/.sqlx/query-7c5a29de07cbe42326a15d4d7fd9714c20b767508e27368be6de38d7b18a3a4d.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id\n FROM alerts\n WHERE workspace_id = $1 AND COALESCE(acknowledged_workspace, false) = $2\n ORDER BY created_at DESC\n LIMIT $3 OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "alert_type", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "message", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "acknowledged", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "workspace_id", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + null, + true + ] + }, + "hash": "7c5a29de07cbe42326a15d4d7fd9714c20b767508e27368be6de38d7b18a3a4d" +} diff --git a/backend/.sqlx/query-a59fae29ebcc9aa53308b777ead3d2b652618ac454f139db738f4499cb4eb079.json b/backend/.sqlx/query-a59fae29ebcc9aa53308b777ead3d2b652618ac454f139db738f4499cb4eb079.json new file mode 100644 index 0000000000..f12fe52c2b --- /dev/null +++ b/backend/.sqlx/query-a59fae29ebcc9aa53308b777ead3d2b652618ac454f139db738f4499cb4eb079.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE alerts SET acknowledged = true", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a59fae29ebcc9aa53308b777ead3d2b652618ac454f139db738f4499cb4eb079" +} diff --git a/backend/.sqlx/query-a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d.json b/backend/.sqlx/query-a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d.json deleted file mode 100644 index 53ef15ffa5..0000000000 --- a/backend/.sqlx/query-a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE alerts SET acknowledged = true WHERE acknowledged = false", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d" -} diff --git a/backend/.sqlx/query-be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580.json b/backend/.sqlx/query-be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580.json deleted file mode 100644 index e726f11fe2..0000000000 --- a/backend/.sqlx/query-be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE alerts SET acknowledged = true WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4" - ] - }, - "nullable": [] - }, - "hash": "be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580" -} diff --git a/backend/.sqlx/query-e05dbe046e846c092a96b6b0a9d872c721169d418586be2d15cee6b929049098.json b/backend/.sqlx/query-e05dbe046e846c092a96b6b0a9d872c721169d418586be2d15cee6b929049098.json new file mode 100644 index 0000000000..c0875942d3 --- /dev/null +++ b/backend/.sqlx/query-e05dbe046e846c092a96b6b0a9d872c721169d418586be2d15cee6b929049098.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT mute_critical_alerts FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "mute_critical_alerts", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "e05dbe046e846c092a96b6b0a9d872c721169d418586be2d15cee6b929049098" +} diff --git a/backend/.sqlx/query-f22168826350797e88153b65a5b1e906a7868f34c062f154b2ee4f24c57aa5c3.json b/backend/.sqlx/query-f22168826350797e88153b65a5b1e906a7868f34c062f154b2ee4f24c57aa5c3.json new file mode 100644 index 0000000000..17b42cb469 --- /dev/null +++ b/backend/.sqlx/query-f22168826350797e88153b65a5b1e906a7868f34c062f154b2ee4f24c57aa5c3.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET mute_critical_alerts = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f22168826350797e88153b65a5b1e906a7868f34c062f154b2ee4f24c57aa5c3" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 55cf6b995c..caed423d9a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -177895136549cdb7c2f2a26ba3b654d76e60d845 \ No newline at end of file +51dcbf93b0d127af9f33fa346cc63fcd2475d4fa diff --git a/backend/migrations/20241108201445_alert_add_workspace_id.down.sql b/backend/migrations/20241108201445_alert_add_workspace_id.down.sql new file mode 100644 index 0000000000..ff3d6821b5 --- /dev/null +++ b/backend/migrations/20241108201445_alert_add_workspace_id.down.sql @@ -0,0 +1,4 @@ +ALTER TABLE alerts DROP COLUMN workspace_id; +ALTER TABLE alerts DROP COLUMN acknowledged_workspace; +ALTER TABLE alerts DROP COLUMN resource; +ALTER TABLE workspace_settings DROP COLUMN mute_critical_alerts; diff --git a/backend/migrations/20241108201445_alert_add_workspace_id.up.sql b/backend/migrations/20241108201445_alert_add_workspace_id.up.sql new file mode 100644 index 0000000000..17ad1c4fe1 --- /dev/null +++ b/backend/migrations/20241108201445_alert_add_workspace_id.up.sql @@ -0,0 +1,4 @@ +ALTER TABLE alerts ADD COLUMN workspace_id TEXT DEFAULT NULL; +ALTER TABLE alerts ADD COLUMN acknowledged_workspace BOOL DEFAULT NULL; +ALTER TABLE alerts ADD COLUMN resource TEXT DEFAULT NULL; +ALTER TABLE workspace_settings ADD COLUMN mute_critical_alerts BOOL DEFAULT NULL; \ No newline at end of file diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index b40c89c766..43c5867dce 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -34,12 +34,12 @@ use windmill_common::{ error, flow_status::FlowStatusModule, global_settings::{ - BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, - DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, - KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, - PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, + JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, + OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, @@ -54,8 +54,8 @@ use windmill_common::{ update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG, WORKER_CONFIG, WORKER_GROUP, }, - BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, - METRICS_DEBUG_ENABLED, METRICS_ENABLED, CRITICAL_ALERT_MUTE_UI_ENABLED + BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, + HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, }; use windmill_queue::cancel_job; use windmill_worker::{ @@ -231,13 +231,20 @@ pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> { } pub async fn reload_critical_alert_mute_ui_setting(db: &DB) -> error::Result<()> { - let mute = load_value_from_global_settings(db, CRITICAL_ALERT_MUTE_UI_SETTING).await; - match mute { - Ok(Some(serde_json::Value::Bool(t))) => { - CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed); + if let Ok(Some(serde_json::Value::Bool(t))) = + load_value_from_global_settings(db, CRITICAL_ALERT_MUTE_UI_SETTING).await + { + CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed); + + if t { + if let Err(e) = sqlx::query!("UPDATE alerts SET acknowledged = true") + .execute(db) + .await + { + tracing::error!("Error updating alerts: {}", e.to_string()); + } } - _ => (), - }; + } Ok(()) } @@ -1323,7 +1330,7 @@ async fn handle_zombie_jobs ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE job_logs.job_id = $1", r.id) .execute(db).await; tracing::error!(error_message); - report_critical_error(error_message, db.clone()).await; + report_critical_error(error_message, db.clone(), Some(&r.workspace_id), None).await; } } @@ -1437,7 +1444,7 @@ async fn handle_zombie_flows( flow.id, flow.workspace_id ); tracing::error!(error_message); - report_critical_error(error_message, db.clone()).await; + report_critical_error(error_message, db.clone(), Some(&flow.workspace_id), None).await; // if the flow hasn't started and is a zombie, we can simply restart it sqlx::query!( "UPDATE queue SET running = false, started_at = null WHERE id = $1 AND canceled = false", @@ -1457,7 +1464,7 @@ async fn handle_zombie_flows( format!("Flow {id} was cancelled because it") } ); - report_critical_error(reason.clone(), db.clone()).await; + report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await; cancel_zombie_flow_job(db, flow, &rsmq, reason).await?; } } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 9d99cf960d..fb0e977a80 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1701,6 +1701,8 @@ paths: type: string default_scripts: $ref: "#/components/schemas/WorkspaceDefaultScripts" + mute_critical_alerts: + type: boolean required: - code_completion_enabled - automatic_billing @@ -2717,6 +2719,112 @@ paths: items: $ref: "#/components/schemas/ContextualVariable" + /w/{workspace}/workspaces/critical_alerts: + get: + summary: Get all critical alerts for this workspace + operationId: workspaceGetCriticalAlerts + tags: + - setting + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - in: query + name: page + schema: + type: integer + default: 1 + description: The page number to retrieve (minimum value is 1) + - in: query + name: page_size + schema: + type: integer + default: 10 + maximum: 100 + description: Number of alerts per page (maximum is 100) + - in: query + name: acknowledged + schema: + type: boolean + nullable: true + description: Filter by acknowledgment status; true for acknowledged, false for unacknowledged, and omit for all alerts + responses: + "200": + description: Successfully retrieved all critical alerts + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/CriticalAlert' + + /w/{workspace}/workspaces/critical_alerts/{id}/acknowledge: + post: + summary: Acknowledge a critical alert for this workspace + operationId: workspaceAcknowledgeCriticalAlert + tags: + - setting + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - in: path + name: id + required: true + schema: + type: integer + description: The ID of the critical alert to acknowledge + responses: + "200": + description: Successfully acknowledged the critical alert + content: + application/json: + schema: + type: string + example: "Critical alert acknowledged" + + /w/{workspace}/workspaces/critical_alerts/acknowledge_all: + post: + summary: Acknowledge all unacknowledged critical alerts for this workspace + operationId: workspaceAcknowledgeAllCriticalAlerts + tags: + - setting + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: Successfully acknowledged all unacknowledged critical alerts. + content: + application/json: + schema: + type: string + example: "All unacknowledged critical alerts acknowledged" + + /w/{workspace}/workspaces/critical_alerts/mute: + post: + summary: Mute critical alert UI for this workspace + operationId: workspaceMuteCriticalAlertsUI + tags: + - setting + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: Boolean flag to mute critical alerts. + required: true + content: + application/json: + schema: + type: object + properties: + mute_critical_alerts: + type: boolean + description: Whether critical alerts should be muted. + example: true + responses: + '200': + description: Successfully updated mute critical alert settings. + content: + application/json: + schema: + type: string + example: "Updated mute critical alert UI settings for workspace: workspace_id" + /oauth/login_callback/{client_name}: post: security: [] @@ -12945,3 +13053,7 @@ components: type: boolean nullable: true description: Acknowledgment status of the alert, can be true, false, or null if not set + workspace_id: + type: string + nullable: true + description: Workspace id if the alert is in the scope of a workspace diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 5dc46e4382..673acb132c 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -434,67 +434,15 @@ pub async fn test_critical_channels() -> Result { Ok("Critical channels require EE".to_string()) } -use serde::Serialize; - -#[derive(Serialize)] -pub struct CriticalAlert { - id: i32, - alert_type: String, - message: String, - created_at: chrono::DateTime, - acknowledged: Option, -} - -#[cfg(feature = "enterprise")] -#[derive(Deserialize)] -pub struct AlertQueryParams { - pub page: Option, - pub page_size: Option, - pub acknowledged: Option, -} - #[cfg(feature = "enterprise")] pub async fn get_critical_alerts( Extension(db): Extension, authed: ApiAuthed, - Query(params): Query, -) -> JsonResult> { + Query(params): Query, +) -> JsonResult> { require_super_admin(&db, &authed.email).await?; - // Default pagination values if not provided - let page = params.page.unwrap_or(1).max(1); - let page_size = params.page_size.unwrap_or(10).min(100) as i64; - let offset = ((page - 1) * page_size as i32) as i64; - - let alerts = if let Some(acknowledged) = params.acknowledged { - sqlx::query_as!( - CriticalAlert, - "SELECT id, alert_type, message, created_at, acknowledged - FROM alerts - WHERE acknowledged = $1 - ORDER BY created_at DESC - LIMIT $2 OFFSET $3", - acknowledged, - page_size, - offset - ) - .fetch_all(&db) - .await? - } else { - sqlx::query_as!( - CriticalAlert, - "SELECT id, alert_type, message, created_at, acknowledged - FROM alerts - ORDER BY created_at DESC - LIMIT $1 OFFSET $2", - page_size, - offset - ) - .fetch_all(&db) - .await? - }; - - Ok(Json(alerts)) + crate::utils::get_critical_alerts(db, params, None).await } #[cfg(not(feature = "enterprise"))] @@ -510,15 +458,7 @@ pub async fn acknowledge_critical_alert( ) -> error::Result { require_super_admin(&db, &authed.email).await?; - sqlx::query!( - "UPDATE alerts SET acknowledged = true WHERE id = $1", - id - ) - .execute(&db) - .await?; - - tracing::info!("Acknowledged critical alert with id: {}", id); - Ok("Critical alert acknowledged".to_string()) + crate::utils::acknowledge_critical_alert(db, None, id).await } #[cfg(not(feature = "enterprise"))] @@ -533,14 +473,7 @@ pub async fn acknowledge_all_critical_alerts( ) -> error::Result { require_super_admin(&db, &authed.email).await?; - sqlx::query!( - "UPDATE alerts SET acknowledged = true WHERE acknowledged = false" - ) - .execute(&db) - .await?; - - tracing::info!("Acknowledged all unacknowledged critical alerts"); - Ok("All unacknowledged critical alerts acknowledged".to_string()) + crate::utils::acknowledge_all_critical_alerts(db, None).await } #[cfg(not(feature = "enterprise"))] diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index ea4fccc160..4302189c4e 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -16,6 +16,12 @@ use windmill_common::{ DB, }; +#[cfg(feature = "enterprise")] +use windmill_common::error::JsonResult; + +#[cfg(feature = "enterprise")] +use axum::Json; + #[derive(Deserialize)] pub struct WithStarredInfoQuery { pub with_starred_info: Option, @@ -162,3 +168,159 @@ pub fn content_plain(body: Body) -> Response { .body(body) .unwrap() } + +use serde::Serialize; + +#[derive(Serialize)] +pub struct CriticalAlert { + id: i32, + alert_type: String, + message: String, + created_at: chrono::DateTime, + acknowledged: Option, + workspace_id: Option, +} + +#[cfg(feature = "enterprise")] +#[derive(Deserialize, Debug)] +pub struct AlertQueryParams { + pub page: Option, + pub page_size: Option, + pub acknowledged: Option, +} + +#[cfg(feature = "enterprise")] +pub async fn get_critical_alerts( + db: DB, + params: AlertQueryParams, + workspace_id: Option, +) -> JsonResult> { + let page = params.page.unwrap_or(1).max(1); + let page_size = params.page_size.unwrap_or(10).min(100) as i64; + let offset = ((page - 1) * page_size as i32) as i64; + + let alerts = if let Some(workspace_id) = workspace_id { + // `workspace_id` is provided => workspace admin + if params.acknowledged.is_none() { + // Case: return all rows where `workspace_id` matches + sqlx::query_as!( + CriticalAlert, + "SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id + FROM alerts + WHERE workspace_id = $1 + ORDER BY created_at DESC + LIMIT $2 OFFSET $3", + workspace_id, + page_size, + offset + ) + .fetch_all(&db) + .await? + } else { + // Case: return rows where `acknowledged_workspace` matches `params.acknowledged` + sqlx::query_as!( + CriticalAlert, + "SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id + FROM alerts + WHERE workspace_id = $1 AND COALESCE(acknowledged_workspace, false) = $2 + ORDER BY created_at DESC + LIMIT $3 OFFSET $4", + workspace_id, + params.acknowledged, + page_size, + offset + ) + .fetch_all(&db) + .await? + } + } else { + // `workspace_id` is not provided => superadmin + if params.acknowledged.is_none() { + // Case: Return all rows unfiltered with global acknowledged as acknowledged + sqlx::query_as!( + CriticalAlert, + "SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id + FROM alerts + ORDER BY created_at DESC + LIMIT $1 OFFSET $2", + page_size, + offset + ) + .fetch_all(&db) + .await? + } else { + // Case: Return rows where global acknowledged matches params.acknowledged + sqlx::query_as!( + CriticalAlert, + "SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id + FROM alerts + WHERE COALESCE(acknowledged, false) = $1 + ORDER BY created_at DESC + LIMIT $2 OFFSET $3", + params.acknowledged, + page_size, + offset + ) + .fetch_all(&db) + .await? + } + }; + + Ok(Json(alerts)) +} + +#[cfg(feature = "enterprise")] +pub async fn acknowledge_critical_alert( + db: DB, + workspace_id: Option, + id: i32, +) -> error::Result { + sqlx::query!( + "UPDATE alerts + SET + acknowledged = true, + acknowledged_workspace = CASE + WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true + ELSE acknowledged_workspace + END + WHERE id = $1", + id, + workspace_id + ) + .execute(&db) + .await?; + + tracing::info!( + "Acknowledged critical alert with id: {}{}", + id, + workspace_id.map_or_else(|| "".to_string(), |w| format!(" for workspace_id: {}", w)) + ); + Ok("Critical alert acknowledged".to_string()) +} + +#[cfg(feature = "enterprise")] +pub async fn acknowledge_all_critical_alerts( + db: DB, + workspace_id: Option, +) -> error::Result { + sqlx::query!( + "UPDATE alerts + SET + acknowledged = true, + acknowledged_workspace = CASE + WHEN $1::text IS NOT NULL THEN true + ELSE acknowledged_workspace + END + WHERE ($1::text IS NOT NULL AND workspace_id = $1) + OR ($1::text IS NULL)", + workspace_id + ) + .execute(&db) + .await?; + + tracing::info!( + "Acknowledged all unacknowledged critical alerts{}", + workspace_id.map_or_else(|| "".to_string(), |w| format!(" for workspace_id: {}", w)) + ); + Ok("All unacknowledged critical alerts acknowledged".to_string()) +} diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs index e10cd94c78..ecdc990061 100644 --- a/backend/windmill-api/src/websocket_triggers.rs +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -754,12 +754,14 @@ async fn disable_with_error(db: &DB, ws_trigger: &WebsocketTrigger, error: Strin ) .execute(db).await { Ok(_) => { - report_critical_error(format!("Disabling websocket {} because of error: {}", ws_trigger.url, error), db.clone()).await; + report_critical_error(format!("Disabling websocket {} because of error: {}", ws_trigger.url, error), db.clone(), Some(&ws_trigger.workspace_id), None).await; }, Err(disable_err) => { report_critical_error( format!("Could not disable websocket {} with err {}, disabling because of error {}", ws_trigger.path, disable_err, error), - db.clone() + db.clone(), + Some(&ws_trigger.workspace_id), + None, ).await; } } @@ -896,7 +898,7 @@ async fn listen_to_websocket( } if should_handle { if let Err(err) = run_job(&db, rsmq.clone(), &ws_trigger, text).await { - report_critical_error(format!("Failed to trigger job from websocket {}: {:?}", ws_trigger.url, err), db.clone()).await; + report_critical_error(format!("Failed to trigger job from websocket {}: {:?}", ws_trigger.url, err), db.clone(), Some(&ws_trigger.workspace_id), None).await; }; } }, diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index ddd227ecbb..36bef54655 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -119,7 +119,11 @@ pub fn workspaced_service() -> Router { .route("/change_workspace_name", post(change_workspace_name)) .route("/change_workspace_id", post(change_workspace_id)) .route("/usage", get(get_usage)) - .route("/used_triggers", get(get_used_triggers)); + .route("/used_triggers", get(get_used_triggers)) + .route("/critical_alerts", get(get_critical_alerts)) + .route("/critical_alerts/:id/acknowledge", post(acknowledge_critical_alert)) + .route("/critical_alerts/acknowledge_all", post(acknowledge_all_critical_alerts)) + .route("/critical_alerts/mute", post(mute_critical_alerts)); #[cfg(feature = "stripe")] { @@ -180,6 +184,7 @@ pub struct WorkspaceSettings { pub default_app: Option, pub automatic_billing: bool, pub default_scripts: Option, + pub mute_critical_alerts: Option, } #[derive(FromRow, Serialize, Debug)] @@ -3075,3 +3080,93 @@ async fn get_usage(Extension(db): Extension, Path(w_id): Path) -> Re .unwrap_or(0); Ok(usage.to_string()) } + +#[cfg(feature = "enterprise")] +pub async fn get_critical_alerts( + Extension(db): Extension, + Path(w_id): Path, + authed: ApiAuthed, + Query(params): Query, +) -> JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + + crate::utils::get_critical_alerts(db, params, Some(w_id)).await +} + +#[cfg(not(feature = "enterprise"))] +pub async fn get_critical_alerts() -> Error { + Error::NotFound("Critical Alerts require EE".to_string()) +} + +#[cfg(feature = "enterprise")] +pub async fn acknowledge_critical_alert( + Extension(db): Extension, + Path((w_id, id)): Path<(String, i32)>, + authed: ApiAuthed, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + crate::utils::acknowledge_critical_alert(db, Some(w_id), id).await +} + +#[cfg(not(feature = "enterprise"))] +pub async fn acknowledge_critical_alert() -> Error { + Error::NotFound("Critical Alerts require EE".to_string()) +} + +#[cfg(feature = "enterprise")] +pub async fn acknowledge_all_critical_alerts( + Extension(db): Extension, + Path(w_id): Path, + authed: ApiAuthed, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + crate::utils::acknowledge_all_critical_alerts(db, Some(w_id)).await +} + +#[cfg(not(feature = "enterprise"))] +pub async fn acknowledge_all_critical_alerts() -> Error { + Error::NotFound("Critical Alerts require EE".to_string()) +} + + +#[cfg(feature = "enterprise")] +#[derive(Deserialize)] +pub struct MuteCriticalAlertRequest { + pub mute_critical_alerts: Option, +} + +#[cfg(feature = "enterprise")] +async fn mute_critical_alerts( + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(m_r): Json, +) -> Result { + require_admin(is_admin, &username)?; + + let mute_alerts = m_r.mute_critical_alerts.unwrap_or(false); + + if mute_alerts { + sqlx::query!( + "UPDATE alerts SET acknowledged_workspace = true, acknowledged = true WHERE workspace_id = $1", + &w_id + ) + .execute(&db) + .await?; + } + + sqlx::query!( + "UPDATE workspace_settings SET mute_critical_alerts = $1 WHERE workspace_id = $2", + mute_alerts, + &w_id + ) + .execute(&db) + .await?; + + Ok(format!("Updated mute criticital alert ui settings for workspace: {}", &w_id)) +} + +#[cfg(not(feature = "enterprise"))] +pub async fn mute_critical_alerts() -> Error { + Error::NotFound("Critical Alerts require EE".to_string()) +} \ No newline at end of file diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 3223ffb75f..1454e8d9ed 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -29,8 +29,11 @@ pub const DEFAULT_PER_PAGE: usize = 1000; pub const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); -use std::sync::atomic::Ordering; use crate::CRITICAL_ALERT_MUTE_UI_ENABLED; +use std::sync::atomic::Ordering; + +#[cfg(feature = "enterprise")] +use crate::worker::CLOUD_HOSTED; lazy_static::lazy_static! { pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new() @@ -249,15 +252,40 @@ pub fn generate_lock_id(database_name: &str) -> i64 { 0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64) } -pub async fn report_critical_error(error_message: String, _db: DB) -> () { +pub async fn report_critical_error( + error_message: String, + _db: DB, + workspace_id: Option<&str>, + resource: Option<&str>, +) -> () { tracing::error!("CRITICAL ERROR: {error_message}"); - let mute = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed); + let mute_global = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed); + let mute_workspace = if let Some(workspace_id) = workspace_id { + match fetch_mute_workspace(&_db, workspace_id).await { + Ok(flag) => flag, + Err(err) => { + tracing::error!("Error fetching mute_workspace: {}", err); + false + } + } + } else { + false + }; + + // we ack_global if mute_global is true, or if mute_workspace is true + // but we ignore global mute setting for ack_workspace + let acknowledge_workspace = mute_workspace; + let acknowledge_global = mute_global || mute_workspace; if let Err(err) = sqlx::query!( - "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('critical_error', $1, $2)", + "INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource) + VALUES ('critical_error', $1, $2, $3, $4, $5)", error_message, - mute + acknowledge_global, + acknowledge_workspace, + workspace_id, + resource, ) .execute(&_db) .await @@ -266,30 +294,86 @@ pub async fn report_critical_error(error_message: String, _db: DB) -> () { } #[cfg(feature = "enterprise")] - send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError, None).await; + if *CLOUD_HOSTED && workspace_id.is_some() { + tracing::error!(error_message) + } else { + send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError, None).await; + } } -pub async fn report_recovered_critical_error(message: String, _db: DB) -> () { +pub async fn report_recovered_critical_error( + message: String, + _db: DB, + workspace_id: Option<&str>, + resource: Option<&str>, +) -> () { tracing::info!("RECOVERED CRITICAL ERROR: {message}"); - let mute = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed); - if let Err(err) = sqlx::query!( - "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('recovered_critical_error', $1, $2)", + "INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource) + VALUES ('recovered_critical_error', $1, $2, $3, $4, $5)", message, - mute + true, + true, + workspace_id, + resource, ) .execute(&_db) .await { - tracing::error!("Failed to save critical error to database: {}", err); + tracing::error!("Failed to save recovered critical error to database: {}", err); } + + // acknowledge all alerts with the same resource + if let Some(resource) = resource { + if let Err(err) = sqlx::query!( + "UPDATE alerts SET acknowledged = true, acknowledged_workspace = true WHERE resource = $1 AND alert_type = 'critical_error'", + resource, + ) + .execute(&_db) + .await + { + tracing::error!("Failed to acknowledge critical error alerts for resource {}: {}", resource, err); + } + } + #[cfg(feature = "enterprise")] - send_critical_alert( - message, - &_db, - CriticalAlertKind::RecoveredCriticalError, - None, + if *CLOUD_HOSTED && workspace_id.is_some() { + tracing::error!(message); + } else { + send_critical_alert( + message, + &_db, + CriticalAlertKind::RecoveredCriticalError, + None, + ) + .await; + } +} + +pub async fn fetch_mute_workspace(_db: &DB, workspace_id: &str) -> Result { + match sqlx::query!( + "SELECT mute_critical_alerts FROM workspace_settings WHERE workspace_id = $1", + workspace_id ) - .await; + .fetch_optional(_db) + .await + { + Ok(Some(record)) => Ok(record.mute_critical_alerts.unwrap_or(false)), + Ok(None) => { + tracing::warn!( + "Workspace ID {} not found in workspace_settings table", + workspace_id + ); + Ok(false) + } + Err(err) => { + tracing::error!( + "Error querying workspace_settings for workspace_id {}: {}", + workspace_id, + err + ); + Err(Error::SqlErr(err)) + } + } } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index dd91226c7e..20122fd878 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -924,6 +924,8 @@ pub async fn add_completed_job< .unwrap_or("".to_string()), ), db.clone(), + Some(&w_id), + None, ) .await; } else if queued_job.email == SCHEDULE_ERROR_HANDLER_USER_EMAIL { @@ -992,7 +994,7 @@ pub async fn add_completed_job< "Could not push workspace error handler for failed job ({base_url}/run/{}?workspace={w_id}): {}", queued_job.id, err - ), db.clone()) + ), db.clone(), Some(&w_id), None) .await; } } @@ -1175,10 +1177,10 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel< queued_job.id, err ); - report_critical_error(error_message, db.clone()).await; + report_critical_error(error_message, db.clone(), Some(&w_id), None).await; } } else { - report_critical_error(error_message, db.clone()).await; + report_critical_error(error_message, db.clone(), Some(&w_id), None).await; } } diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index f9b2c90151..213318e6fe 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -14,7 +14,7 @@ import Alert from './common/alert/Alert.svelte' import { isCloudHosted } from '$lib/cloud' import { capitalize, classNames, sleep } from '$lib/utils' - import { enterpriseLicense } from '$lib/stores' + import { enterpriseLicense, isCriticalAlertsUIOpen } from '$lib/stores' import CustomOauth from './CustomOauth.svelte' import { AlertCircle, @@ -43,6 +43,7 @@ export let tab: string = 'Core' export let hideTabs: boolean = false export let hideSave: boolean = false + export let closeDrawer: (() => void) | undefined = () => {} let values: Record = {} let initialOauths: Record = {} @@ -814,6 +815,26 @@ placeholder={setting.placeholder} bind:value={values[setting.key]} /> + {:else if setting.key == 'critical_alert_mute_ui'} +
+ +
+ +
+
{:else if setting.fieldType == 'critical_error_channels'}
diff --git a/frontend/src/lib/components/SuperadminSettings.svelte b/frontend/src/lib/components/SuperadminSettings.svelte index e47466a02a..f5e88116f6 100644 --- a/frontend/src/lib/components/SuperadminSettings.svelte +++ b/frontend/src/lib/components/SuperadminSettings.svelte @@ -325,7 +325,7 @@
- + diff --git a/frontend/src/lib/components/common/modal/Modal.svelte b/frontend/src/lib/components/common/modal/Modal.svelte index 718a52000a..300499d71a 100644 --- a/frontend/src/lib/components/common/modal/Modal.svelte +++ b/frontend/src/lib/components/common/modal/Modal.svelte @@ -44,7 +44,7 @@
(open = false)} transition:fadeFast|local - class={'absolute top-0 bottom-0 left-0 right-0 z-50'} + class={'absolute top-0 bottom-0 left-0 right-0 z-[9999]'} role="dialog" >
= { { label: 'Critical alert channels', description: - 'Channels to send critical alerts to. SMTP must be configured for the email channel. A Slack workspace must be connected to the instance for the Slack channel. Learn more', + 'Channels to send critical alerts to. SMTP must be configured for the email channel. A Slack workspace must be connected to the instance for the Slack channel. Learn more', key: 'critical_error_channels', fieldType: 'critical_error_channels', storage: 'setting', diff --git a/frontend/src/lib/components/sidebar/CriticalAlertButton.svelte b/frontend/src/lib/components/sidebar/CriticalAlertButton.svelte deleted file mode 100644 index 9c6d3db68d..0000000000 --- a/frontend/src/lib/components/sidebar/CriticalAlertButton.svelte +++ /dev/null @@ -1,64 +0,0 @@ - - -{#if !disabled} - -
- -
- - {#if label} - {label} - {/if} - - -{/if} diff --git a/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte b/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte index 68d7bfbffb..249deecccf 100644 --- a/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte +++ b/frontend/src/lib/components/sidebar/CriticalAlertModal.svelte @@ -3,16 +3,62 @@ import CriticalAlertModalInner from './CriticalAlertModalInner.svelte' import { SettingService } from '$lib/gen' import { sendUserToast } from '$lib/toast' + import { superadmin, workspaceStore, isCriticalAlertsUIOpen } from '$lib/stores' import Modal from '../common/modal/Modal.svelte' export let open: boolean = false export let numUnacknowledgedCriticalAlerts: number = 0 + export let muteSettings + let workspaceContext = false + + $: { + setupApiFunctions(workspaceContext) + } + + function setupApiFunctions(_ctx?) { + getCriticalAlerts = withSuperadminLogic( + SettingService.getCriticalAlerts, + SettingService.workspaceGetCriticalAlerts + ) + + acknowledgeCriticalAlert = withSuperadminLogic( + SettingService.acknowledgeCriticalAlert, + SettingService.workspaceAcknowledgeCriticalAlert + ) + + acknowledgeAllCriticalAlerts = withSuperadminLogic( + SettingService.acknowledgeAllCriticalAlerts, + SettingService.workspaceAcknowledgeAllCriticalAlerts + ) + } + + $: isCriticalAlertsUIOpen.set(open) + $: if ($isCriticalAlertsUIOpen) open = $isCriticalAlertsUIOpen let checkForNewAlertsInterval: ReturnType let checkingForNewAlerts = false - onMount(() => { - updateHasUnacknowledgedCriticalAlerts(true) + const withSuperadminLogic = (superadminFunction, workspaceFunction) => { + return async (params = {}) => { + if (!$superadmin || workspaceContext) { + return workspaceFunction({ + ...params, + workspace: $workspaceStore + }) + } else { + return superadminFunction(params) + } + } + } + + let getCriticalAlerts + let acknowledgeCriticalAlert + let acknowledgeAllCriticalAlerts + + setupApiFunctions() + + onMount(async () => { + await updateHasUnacknowledgedCriticalAlerts(false) checkForNewAlertsInterval = setInterval(() => { updateHasUnacknowledgedCriticalAlerts(true) }, 15000) @@ -25,15 +71,18 @@ async function updateHasUnacknowledgedCriticalAlerts(sendToast: boolean = false) { if (checkingForNewAlerts) return checkingForNewAlerts = true - try { - const unacknowledged = await SettingService.getCriticalAlerts({ + const unacknowledged = await getCriticalAlerts({ page: 1, pageSize: 10, acknowledged: false }) - - if (numUnacknowledgedCriticalAlerts === 0 && unacknowledged.length > 0 && sendToast) { + if ( + numUnacknowledgedCriticalAlerts === 0 && + unacknowledged.length > 0 && + sendToast && + (($superadmin && !muteSettings.global) || (!$superadmin && !muteSettings.workspace)) + ) { sendUserToast( 'Critical Alert:', true, @@ -62,11 +111,19 @@ } async function acknowledgeAlert(id: number) { - await SettingService.acknowledgeCriticalAlert({ id }) + await acknowledgeCriticalAlert({ id }) updateHasUnacknowledgedCriticalAlerts() } - + diff --git a/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte b/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte index 34ff0ffa35..5b9a53992c 100644 --- a/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte +++ b/frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte @@ -2,23 +2,77 @@ import Button from '../common/button/Button.svelte' import Toggle from '$lib/components/Toggle.svelte' import { SettingService } from '$lib/gen' - import { CheckCircle2, AlertCircle, RefreshCw, CheckSquare2, AlertTriangle } from 'lucide-svelte' + import { + CheckCircle2, + AlertCircle, + RefreshCw, + CheckSquare2, + AlertTriangle, + } from 'lucide-svelte' import type { CriticalAlert } from '$lib/gen' import { onMount } from 'svelte' import { instanceSettingsSelectedTab } from '$lib/stores' import { goto } from '$app/navigation' + import { superadmin, workspaceStore } from '$lib/stores' + import { sendUserToast } from '$lib/toast' + import Section from '$lib/components/Section.svelte' - export let updateHasUnacknowledgedCriticalAlerts: () => void = () => {} + export let updateHasUnacknowledgedCriticalAlerts + export let getCriticalAlerts + export let acknowledgeCriticalAlert + export let acknowledgeAllCriticalAlerts + export let numUnacknowledgedCriticalAlerts let alerts: CriticalAlert[] = [] let isRefreshing = false - let hasCriticalAlertChannels = false + let hasCriticalAlertChannels = true + + export let muteSettings = { + workspace: true, + global: true + } + + $: muteSettings + $: { + if (initialMuteSettings.workspace !== muteSettings.workspace || initialMuteSettings.global !== muteSettings.global) { + saveMuteSettings() + } + } + + $: numUnacknowledgedCriticalAlerts >= 0 && getAlerts(true) + + let initialMuteSettings = muteSettings + + async function saveMuteSettings() { + if (initialMuteSettings.workspace !== muteSettings.workspace) { + // Workspace + await SettingService.workspaceMuteCriticalAlertsUi({ + workspace: $workspaceStore!, + requestBody: { + mute_critical_alerts: muteSettings.workspace + } + }) + } + if ($superadmin && initialMuteSettings.global !== muteSettings.global) { + // Global + await SettingService.setGlobal({ + key: 'critical_alert_mute_ui', + requestBody: { value: muteSettings.global } + }) + } + sendUserToast( + `Critical alert UI mute settings changed.\nPlease reload page for UI changes to take effect.` + ) + getAlerts(true) + initialMuteSettings = { ...muteSettings } + } $: loading = isRefreshing onMount(() => { refreshAlerts() + initialMuteSettings = { ...muteSettings } }) // Pagination @@ -29,14 +83,14 @@ let hideAcknowledged = false async function acknowledgeAll() { - await SettingService.acknowledgeAllCriticalAlerts() + await acknowledgeAllCriticalAlerts() getAlerts(false) } async function fetchAlerts(pageNumber: number) { isRefreshing = true try { - const newAlerts = await SettingService.getCriticalAlerts({ + const newAlerts = await getCriticalAlerts({ page: pageNumber, pageSize: pageSize, acknowledged: hideAcknowledged ? false : undefined @@ -55,6 +109,7 @@ async function getAlerts(reset?: boolean) { if (reset) page = 1 + updateHasUnacknowledgedCriticalAlerts() await fetchAlerts(page) } @@ -64,7 +119,7 @@ } async function acknowledgeAlert(id: number) { - await SettingService.acknowledgeCriticalAlert({ id }) + await acknowledgeCriticalAlert({ id }) getAlerts(false) } @@ -82,7 +137,7 @@ } async function refreshAlerts() { - checkCriticalAlertChannels() + if ($superadmin) checkCriticalAlertChannels() await getAlerts(true) } @@ -102,46 +157,99 @@ goto('/#superadmin-settings') instanceSettingsSelectedTab.set('Core') } + + export let workspaceContext = false + + $: { + workspaceContextChanged(workspaceContext) + } + + async function workspaceContextChanged(_ctx) { + await getAlerts(true) + }
- {#if !hasCriticalAlertChannels} -
- -

- No critical alert channels are set up. Go to the - Instance Settings - page to configure critical alert channels. -

+
+
+ {#if !hasCriticalAlertChannels && $superadmin} +
+ +

+ No critical alert channels are set up. Go to the + Instance Settings + page to configure critical alert channels. +

+
+ {/if}
- {/if} - -
-
- +
+ +
+ + {#if $superadmin} +
+ +
+ {/if} + +
-
+ {#if $superadmin} +
+ +
+ {/if} - -
+
+ +
+ - -
-
- - Page {page} - -
-
- +
+
+ +
+ +
@@ -153,11 +261,14 @@ Type Message Created At + {#if $superadmin} + Workspace + {/if} Acknowledge - {#each alerts as { id, alert_type, message, created_at, acknowledged }} + {#each alerts as { id, alert_type, message, created_at, acknowledged, workspace_id }} {#if !hideAcknowledged || !acknowledged} @@ -174,6 +285,9 @@ {message} {formatDate(created_at)} + {#if $superadmin} + {workspace_id ? workspace_id : 'global'} + {/if}
{#if !acknowledged} @@ -196,6 +310,11 @@
+
+ + Page {page} + +
{#if alerts.length === 0}

No critical alerts available.

diff --git a/frontend/src/lib/components/sidebar/MenuButton.svelte b/frontend/src/lib/components/sidebar/MenuButton.svelte index 35cd762332..5b3c048d0e 100644 --- a/frontend/src/lib/components/sidebar/MenuButton.svelte +++ b/frontend/src/lib/components/sidebar/MenuButton.svelte @@ -2,14 +2,15 @@ import { twMerge } from 'tailwind-merge' import Popover from '../Popover.svelte' import { createEventDispatcher } from 'svelte' - + import SideBarNotification from './SideBarNotification.svelte' export let label: string | undefined = undefined export let icon: any | undefined = undefined export let isCollapsed: boolean export let disabled: boolean = false export let lightMode: boolean = false export let stopPropagationOnClick: boolean = false - export let shortcut: string = "" + export let shortcut: string = '' + export let notificationsCount: number = 0 let dispatch = createEventDispatcher() @@ -60,6 +61,16 @@ {/if} + + {#if isCollapsed && notificationsCount > 0} +
+ +
+ {:else if notificationsCount > 0} +
+ +
+ {/if} {#if label} diff --git a/frontend/src/lib/components/sidebar/SideBarNotification.svelte b/frontend/src/lib/components/sidebar/SideBarNotification.svelte new file mode 100644 index 0000000000..d935769972 --- /dev/null +++ b/frontend/src/lib/components/sidebar/SideBarNotification.svelte @@ -0,0 +1,14 @@ + + +{#if !small} +
+ {notificationCount > 9 ? '9+' : notificationCount} +
+{:else} +
+{/if} diff --git a/frontend/src/lib/components/sidebar/SidebarContent.svelte b/frontend/src/lib/components/sidebar/SidebarContent.svelte index 3d45f895ff..4e34071256 100644 --- a/frontend/src/lib/components/sidebar/SidebarContent.svelte +++ b/frontend/src/lib/components/sidebar/SidebarContent.svelte @@ -1,6 +1,13 @@
@@ -435,17 +448,6 @@ {/if}
- {#if $superadmin && $enterpriseLicense} - openCriticalAlertsModal()} - {numUnacknowledgedCriticalAlerts} - {isCollapsed} - label="Critical Alerts" - class="!text-xs" - disabled={numUnacknowledgedCriticalAlerts === 0} - /> - {/if}
@@ -460,7 +462,12 @@ />
- +
- +
diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 71d868779e..6e9ba1d116 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -15,7 +15,13 @@ import Tooltip from '$lib/components/Tooltip.svelte' import WorkspaceUserSettings from '$lib/components/settings/WorkspaceUserSettings.svelte' import { WORKSPACE_SHOW_SLACK_CMD, WORKSPACE_SHOW_WEBHOOK_CLI_SYNC } from '$lib/consts' - import { OauthService, WorkspaceService, JobService, ResourceService } from '$lib/gen' + import { + OauthService, + WorkspaceService, + JobService, + ResourceService, + SettingService + } from '$lib/gen' import { enterpriseLicense, copilotInfo, @@ -23,7 +29,8 @@ userStore, usersWorkspaceStore, workspaceStore, - hubBaseUrlStore + hubBaseUrlStore, + isCriticalAlertsUIOpen } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { emptyString, tryEvery } from '$lib/utils' @@ -102,6 +109,8 @@ let errorHandlerItemKind: 'flow' | 'script' = 'script' let errorHandlerExtraArgs: Record = {} let errorHandlerMutedOnCancel: boolean | undefined = undefined + let criticalAlertUIMuted: boolean | undefined = undefined + let initialCriticalAlertUIMuted: boolean | undefined = undefined let aiResourceInitialPath: string | undefined = undefined let aiResourceInitialProvider: string | undefined = undefined @@ -391,6 +400,8 @@ errorHandlerScriptPath = (settings.error_handler ?? '').split('/').slice(1).join('/') errorHandlerInitialScriptPath = errorHandlerScriptPath errorHandlerMutedOnCancel = settings.error_handler_muted_on_cancel + criticalAlertUIMuted = settings.mute_critical_alerts + initialCriticalAlertUIMuted = settings.mute_critical_alerts if (emptyString($enterpriseLicense)) { errorHandlerSelected = 'custom' } else { @@ -587,6 +598,22 @@ timeout: 5000 }) } + + async function editCriticalAlertMuteSetting() { + await SettingService.workspaceMuteCriticalAlertsUi({ + workspace: $workspaceStore!, + requestBody: { + mute_critical_alerts: criticalAlertUIMuted + } + }) + sendUserToast( + `Critical alert UI mute setting for workspace is set to ${criticalAlertUIMuted}\nreloading page...` + ) + // reload page after change of setting + setTimeout(() => { + window.location.reload() + }, 3000) + } @@ -972,6 +999,41 @@ Save
+
+
+
Workspace Critical Alerts
+
+ Critical alerts within the scope of a workspace are sent to the workspace admins through + a UI notification. + Learn more. +
+
+ + + +
+
+
{:else if tab == 'ai'}
From 44f3dcc2b3dff9c41bacd06f89cd03326c18dd90 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 15 Nov 2024 10:16:52 +0100 Subject: [PATCH 18/31] improve variable and resource not visible error message --- backend/windmill-api/src/resources.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index bb881d5369..543a978fac 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -300,7 +300,7 @@ async fn get_resource( .await?; tx.commit().await?; if resource_o.is_none() { - explain_resource_perm_error(&path, &w_id, &db).await?; + explain_resource_perm_error(&path, &w_id, &db, &authed).await?; } let resource = not_found_if_none(resource_o, "Resource", path)?; Ok(Json(resource)) @@ -343,7 +343,7 @@ async fn get_resource_value( tx.commit().await?; if value_o.is_none() { - explain_resource_perm_error(&path, &w_id, &db).await?; + explain_resource_perm_error(&path, &w_id, &db, &authed).await?; } let value = not_found_if_none(value_o, "Resource", path)?; @@ -354,6 +354,7 @@ async fn explain_resource_perm_error( path: &str, w_id: &str, db: &sqlx::Pool, + authed: &ApiAuthed, ) -> windmill_common::error::Result<()> { let extra_perms = sqlx::query_scalar!( "SELECT extra_perms from resource WHERE path = $1 AND workspace_id = $2", @@ -378,12 +379,12 @@ async fn explain_resource_perm_error( .fetch_optional(db) .await?; return Err(Error::NotAuthorized(format!( - "Resource exists but you don't have access to it:\nresource perms: {}\nfolder perms: {}", + "Resource exists but you don't have access to it:\nresource perms: {}\nfolder perms: {}\nauthed as: {authed:?}", serde_json::to_string_pretty(&extra_perms).unwrap_or_default(), serde_json::to_string_pretty(&folder_extra_perms).unwrap_or_default() ))); } else { return Err(Error::NotAuthorized(format!( - "Resource exists but you don't have access to it:\nresource perms: {}", + "Resource exists but you don't have access to it:\nresource perms: {}\nauthed as: {authed:?}", serde_json::to_string_pretty(&extra_perms).unwrap_or_default() ))); } @@ -457,7 +458,7 @@ pub async fn get_resource_value_interpolated_internal( .await?; tx.commit().await?; if value_o.is_none() { - explain_resource_perm_error(path, workspace, db).await?; + explain_resource_perm_error(path, workspace, db, &authed).await?; } let value = not_found_if_none(value_o, "Resource", path)?; From f7ce4d1c8bb38d4d6a4e2b97fbd3c95093799655 Mon Sep 17 00:00:00 2001 From: Lucas Abel <22837557+uael@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:56:41 +0100 Subject: [PATCH 19/31] benchmarks: reduce 'big' task job count (#4718) --- benchmarks/suite_config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/suite_config.json b/benchmarks/suite_config.json index 7619d4d3b9..a2704a46fe 100644 --- a/benchmarks/suite_config.json +++ b/benchmarks/suite_config.json @@ -33,10 +33,10 @@ }, { "kind": "bigrawscript", - "jobs": 10000 + "jobs": 4000 }, { "kind": "bigscriptinflow", - "jobs": 10000 + "jobs": 4000 } ] \ No newline at end of file From ff5fcc59d481c142df8d27fe36289a7f1e164d61 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 15 Nov 2024 10:31:42 -0500 Subject: [PATCH 20/31] if CLOUD_HOSTED and workspace_id set, autoacknowledge for superadmin (#4719) --- backend/windmill-common/src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 1454e8d9ed..a560eead2f 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -276,7 +276,7 @@ pub async fn report_critical_error( // we ack_global if mute_global is true, or if mute_workspace is true // but we ignore global mute setting for ack_workspace let acknowledge_workspace = mute_workspace; - let acknowledge_global = mute_global || mute_workspace; + let acknowledge_global = mute_global || mute_workspace || ( workspace_id.is_some() && *CLOUD_HOSTED); if let Err(err) = sqlx::query!( "INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource) From 9ce1d46459d1c78fb19766e35afc5e867a4d07bc Mon Sep 17 00:00:00 2001 From: Henri Courdent <122811744+hcourdent@users.noreply.github.com> Date: Fri, 15 Nov 2024 16:55:53 +0100 Subject: [PATCH 21/31] Result node placeholder (#4721) --- .../src/lib/components/flows/content/FlowEditorPanel.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte b/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte index 4da8380fa6..deb9c6dd17 100644 --- a/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte +++ b/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte @@ -62,7 +62,7 @@ {:else if $selectedId === 'Input'} {:else if $selectedId === 'Result'} -

Nothing to show about the result node. Happy flow building!

+

The result of the flow will be the result of the last node.

{:else if $selectedId === 'constants'} {:else if $selectedId === 'failure'} From c069969732797b2469d18be19844831ba9689db7 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 15 Nov 2024 12:14:11 -0500 Subject: [PATCH 22/31] cloud_hosted non ee scope (#4723) --- backend/windmill-common/src/utils.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index a560eead2f..340ecdc0dc 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -32,7 +32,6 @@ pub const GIT_VERSION: &str = use crate::CRITICAL_ALERT_MUTE_UI_ENABLED; use std::sync::atomic::Ordering; -#[cfg(feature = "enterprise")] use crate::worker::CLOUD_HOSTED; lazy_static::lazy_static! { From 9ad1afc8f2d941386c9eaad5894448bd32c1bcbf Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Fri, 15 Nov 2024 18:35:21 +0100 Subject: [PATCH 23/31] unbreak ::text for json in pg (#4722) * unbreak ::text for json in pg * better error handling --- backend/windmill-worker/src/pg_executor.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 55f5abea31..48fee2b267 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -583,6 +583,11 @@ fn convert_val( .unwrap_or(vec![]); Ok(Box::new(bytes)) } + Value::Object(_) if arg_t == "text" || arg_t == "varchar" => { + Ok(Box::new(serde_json::to_string(value).map_err(|err| { + Error::ExecutionErr(format!("Failed to convert JSON to text: {}", err)) + })?)) + } Value::Object(_) => Ok(Box::new(value.clone())), Value::String(s) => Ok(Box::new(s.clone())), _ => Err(Error::ExecutionErr(format!( From 56f70fd7decd05875e81f992890136e4d1573a68 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Fri, 15 Nov 2024 18:24:57 +0000 Subject: [PATCH 24/31] Ignore fields with `false` value in DiffViewer (#4712) * Convert false to undefined * Remove comments * Make it consistent * Formatting * Apply at downstream --- frontend/src/lib/components/DiffDrawer.svelte | 4 +- .../src/lib/components/FlowBuilder.svelte | 30 +++++++------ .../src/lib/components/ScriptBuilder.svelte | 45 ++++++++++++------- .../apps/editor/AppEditorHeader.svelte | 34 +++++++++----- frontend/src/lib/utils.ts | 24 ++++++++++ .../flows/edit/[...path]/+page.svelte | 2 +- 6 files changed, 97 insertions(+), 42 deletions(-) diff --git a/frontend/src/lib/components/DiffDrawer.svelte b/frontend/src/lib/components/DiffDrawer.svelte index 7b461534c9..37979e28d0 100644 --- a/frontend/src/lib/components/DiffDrawer.svelte +++ b/frontend/src/lib/components/DiffDrawer.svelte @@ -8,6 +8,7 @@ cleanValueProperties, orderedJsonStringify, orderedYamlStringify, + replaceFalseWithUndefined, type Value } from '$lib/utils' import type { Script } from '$lib/gen' @@ -54,8 +55,9 @@ diffViewer.closeDrawer() } + function prepareDiff(data: Value) { - const metadata = structuredClone(cleanValueProperties(data)) + const metadata = structuredClone(cleanValueProperties(replaceFalseWithUndefined(data))) const content = metadata['content'] if (metadata['content'] !== undefined) { metadata['content'] = 'check content diff' diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index c90525db4b..1835c4f330 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -25,6 +25,7 @@ encodeState, formatCron, orderedJsonStringify, + replaceFalseWithUndefined, sleep, type Value } from '$lib/utils' @@ -294,12 +295,21 @@ // We need it for diff await syncWithDeployed() - // Handle through confirmation modal - confirmCallback = async () => { + if ( + deployedValue && + $flowStore && + orderedJsonStringify(deployedValue) === + orderedJsonStringify(replaceFalseWithUndefined({ ...$flowStore, path: $pathStore })) + ) { await saveFlow(deploymentMsg) + } else { + // Handle through confirmation modal + confirmCallback = async () => { + await saveFlow(deploymentMsg) + } + // Open confirmation modal + open = true } - // Open confirmation modal - open = true } } async function syncWithDeployed() { @@ -308,18 +318,12 @@ path: initialPath, withStarredInfo: true }) - deployedValue = { + deployedValue = replaceFalseWithUndefined({ ...flow, - starred: undefined, - id: undefined, edited_at: undefined, edited_by: undefined, - workspace_id: undefined, - archived: undefined, - same_worker: undefined, - visible_to_runner_only: undefined, - ws_error_handler_muted: undefined - } + workspace_id: undefined + }) deployedBy = flow.edited_by } diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 44a73f5bdd..4728ae8294 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -18,6 +18,7 @@ encodeState, formatCron, orderedJsonStringify, + replaceFalseWithUndefined, type Value } from '$lib/utils' import Path from './Path.svelte' @@ -278,17 +279,28 @@ // Fetch entire script, since we need it to show Diff await syncWithDeployed() - // Handle through confirmation modal - confirmCallback = async () => { - open = false - if (actual_parent_hash) { - await editScript(stay, actual_parent_hash, deployMsg) - } else { - sendUserToast('Could not fetch latest version of the script', true) + if ( + deployedValue && + script && + orderedJsonStringify({ ...deployedValue, hash: undefined }) === + orderedJsonStringify( + replaceFalseWithUndefined({ ...script, hash: undefined, parent_hash: undefined }) + ) + ) { + await editScript(stay, actual_parent_hash, deployMsg) + } else { + // Handle through confirmation modal + confirmCallback = async () => { + open = false + if (actual_parent_hash) { + await editScript(stay, actual_parent_hash, deployMsg) + } else { + sendUserToast('Could not fetch latest version of the script', true) + } } + // Open confirmation modal + open = true } - // Open confirmation modal - open = true } } @@ -299,20 +311,16 @@ withStarredInfo: true }) - deployedValue = { + deployedValue = replaceFalseWithUndefined({ ...latestScript, - starred: undefined, workspace_id: undefined, - archived: undefined, created_at: undefined, created_by: undefined, - deleted: undefined, extra_perms: undefined, - is_template: undefined, lock: undefined, lock_error_logs: undefined, parent_hashes: undefined - } + }) deployedBy = latestScript.created_by } @@ -691,8 +699,11 @@