mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-18 16:02:10 +00:00
feat: refactor entirely json processing in favor or rawjson to handle larger payloads (#2446)
* raw json * raw json * raw json * raw json * the big rawjson rewrite * resolve merge conflicts * all * progress * progress * all compile * remove mut * remove mut * remove mut * arc queuedjob * finish * finish * finish * fixes * progress * fix result * small fixes * small fixes * fix tests * fix tests * fix tests * optimize eval timeout * optimize eval timeout * optimize eval timeout * handle big args and result in get jobs * improve args * fix * fix * fix tests * fix tests * done * add bigquery to native picker * add bigquery --------- Co-authored-by: gbouv <guillaume@windmill.dev>
This commit is contained in:
Ruben Fiszel
GitHub
@@ -49,7 +49,7 @@ jobs:
|
||||
with:
|
||||
ref: benchmarks
|
||||
- name: benchmark
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 20
|
||||
run: deno run --unstable -A -r
|
||||
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
|
||||
-c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
|
||||
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT mem_peak FROM queue WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "mem_peak",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "0715955b7e98cc669a88eca6556cd46b7f9c07fcf32a24f85b69720b54e6e95f"
|
||||
}
|
||||
+1
@@ -37,6 +37,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT result, id\n FROM completed_job\n WHERE id = ANY($1)\n AND workspace_id = $2\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "result",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "id",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "1f040850c2a82bc09789226b167c43fd4935cfbb4951760a4d527665b70a5ac7"
|
||||
}
|
||||
+1
@@ -67,6 +67,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
+1
@@ -28,6 +28,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT value, approver, resume_id FROM resume_job WHERE job = $1 ORDER BY created_at ASC",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "value",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "approver",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "resume_id",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "541ebd3bac65431237cf3b882dfdcd61ca97c253d9754d05bba59fda89841067"
|
||||
}
|
||||
+1
@@ -42,6 +42,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
+1
@@ -42,6 +42,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
+1
@@ -46,6 +46,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE queue SET mem_peak = GREATEST($1, mem_peak), last_ping = now() WHERE id = $2 RETURNING canceled",
|
||||
"query": "UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -19,5 +19,5 @@
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "913c634de21d46b8841f8a7c25c408da7c572f9e685db6351848cbf6e9253efc"
|
||||
"hash": "8b221f0d08f3304364e56f5c4894fea42975b03d21904655d724a35163413d29"
|
||||
}
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT raw_flow->'modules'->$1::int->'stop_after_if'->>'expr' as stop_early_expr,\n (raw_flow->'modules'->$1::int->'stop_after_if'->>'skip_if_stopped')::bool as skip_if_stopped,\n args \n FROM queue\n WHERE id = $2\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "stop_early_expr",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "skip_if_stopped",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "args",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int4",
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
null,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "8cea673a5b17bc7cf671b539c2c1eb34731c77aaf42dc12e45472eeb134887d0"
|
||||
}
|
||||
+1
@@ -40,6 +40,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
+1
@@ -60,6 +60,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT result FROM completed_job WHERE id = ANY($1) AND workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "result",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "a227548b6604c56bfc15eb780bd8ee72a89dc6701a50f5048e928bd87baa7b9a"
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO resource\n (workspace_id, path, value, resource_type)\n VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "ad9ccde8d831461f1f312d530867714577a825cd40d6655ed61b0b343b5d4482"
|
||||
}
|
||||
+1
@@ -42,6 +42,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT result FROM completed_job WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "result",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "c2849e67b9fea0dc46e6d7000f5a0c9dab89ae80a183d9255f8fdb356b4bc61c"
|
||||
}
|
||||
+9
-3
@@ -1,15 +1,20 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3",
|
||||
"query": "SELECT running, substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "running",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "logs",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"ordinal": 2,
|
||||
"name": "mem_peak",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
@@ -22,9 +27,10 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
null,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "b69f747eae8b13a0a8d6914a3a7ad322554fcfe62cd28c5f0a475f18dd770d61"
|
||||
"hash": "dda02bee4e15e0f3a2b0cb17ca86ae7ee3a4c0413d3484d3bc960796b39cd650"
|
||||
}
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT error_handler FROM workspace_settings WHERE workspace_id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "error_handler",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "f8b34e09453d51d3df5be20652938a890ee353fec64ee73c312a3352da7f7515"
|
||||
}
|
||||
Generated
+2
-1
@@ -7383,6 +7383,7 @@ version = "1.183.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
"axum",
|
||||
"bigdecimal 0.4.1",
|
||||
"chrono",
|
||||
"chrono-tz",
|
||||
@@ -7441,6 +7442,7 @@ dependencies = [
|
||||
"prometheus",
|
||||
"rand 0.8.5",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rsmq_async",
|
||||
"rust_decimal",
|
||||
"serde",
|
||||
@@ -7452,7 +7454,6 @@ dependencies = [
|
||||
"tracing",
|
||||
"urlencoding",
|
||||
"uuid 1.4.1",
|
||||
"windmill-api-client",
|
||||
"windmill-audit",
|
||||
"windmill-common",
|
||||
"windmill-parser",
|
||||
|
||||
+3
-3
@@ -11,7 +11,6 @@ members = [
|
||||
"./windmill-worker",
|
||||
"./windmill-common",
|
||||
"./windmill-audit",
|
||||
"./windmill-api-client",
|
||||
"./parsers/windmill-parser",
|
||||
"./parsers/windmill-parser-ts",
|
||||
"./parsers/windmill-parser-wasm",
|
||||
@@ -45,7 +44,6 @@ tokio.workspace = true
|
||||
dotenv.workspace = true
|
||||
windmill-common = { workspace = true, features = ["tracing_init"] }
|
||||
windmill-api.workspace = true
|
||||
windmill-api-client.workspace = true
|
||||
windmill-worker.workspace = true
|
||||
futures.workspace = true
|
||||
tracing.workspace = true
|
||||
@@ -72,10 +70,11 @@ reqwest.workspace = true
|
||||
windmill-queue.workspace = true
|
||||
axum.workspace = true
|
||||
serde.workspace = true
|
||||
windmill-api-client.workspace = true
|
||||
|
||||
|
||||
[workspace.dependencies]
|
||||
windmill-api = { path = "./windmill-api" }
|
||||
windmill-api-client = { path = "./windmill-api-client" }
|
||||
windmill-queue = { path = "./windmill-queue" }
|
||||
windmill-worker = { path = "./windmill-worker" }
|
||||
windmill-common = { path = "./windmill-common" }
|
||||
@@ -88,6 +87,7 @@ windmill-parser-go = { path = "./parsers/windmill-parser-go" }
|
||||
windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
|
||||
windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
|
||||
windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
|
||||
windmill-api-client = { path = "./windmill-api-client" }
|
||||
|
||||
axum = { version = "^0", features = ["headers"] }
|
||||
headers = "^0"
|
||||
|
||||
+8
-9
@@ -5,15 +5,14 @@ contains files used to build the "root" binary.
|
||||
|
||||
## Components
|
||||
|
||||
| name | description |
|
||||
| --------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
|
||||
| [windmill-api](./windmill-api/) | The API server, exposing functionality to other components and the frontend |
|
||||
| [windmill-api-client](./windmill-api-client/) | An autogenerated Rust API client, used by other components to talk to the API |
|
||||
| [windmill-audit](./windmill-audit/) | Contains audit functionality, allowing different components to record important actions |
|
||||
| [windmill-common](./windmill-common/) | Common code shared by all crates |
|
||||
| [windmill-queue](./windmill-queue/) | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers |
|
||||
| [windmill-worker](./windmill-worker/) | The worker. Used to process and execute flows & jobs. |
|
||||
| [parsers](./parsers/) | Contains code to parse signatures in different langauges. |
|
||||
| name | description |
|
||||
| ------------------------------------- | --------------------------------------------------------------------------------------------------------- |
|
||||
| [windmill-api](./windmill-api/) | The API server, exposing functionality to other components and the frontend |
|
||||
| [windmill-audit](./windmill-audit/) | Contains audit functionality, allowing different components to record important actions |
|
||||
| [windmill-common](./windmill-common/) | Common code shared by all crates |
|
||||
| [windmill-queue](./windmill-queue/) | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers |
|
||||
| [windmill-worker](./windmill-worker/) | The worker. Used to process and execute flows & jobs. |
|
||||
| [parsers](./parsers/) | Contains code to parse signatures in different langauges. |
|
||||
|
||||
### Compile sqlx for offline ci
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
-- Add down migration script here
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Add up migration script here
|
||||
UPDATE config set config = '{"worker_tags": ["nativets", "postgresq", "mysql", "graphql", "snowflake", "bigquery"]}'::jsonb where name = 'worker__native' and config = '{"worker_tags": ["nativets", "postgresql", "mysql", "graphql"
|
||||
, "snowflake"]}'::jsonb ;
|
||||
@@ -1,6 +1,5 @@
|
||||
use std::{collections::HashMap, fmt::Display, ops::Mul, str::FromStr, sync::Arc, time::Duration};
|
||||
|
||||
use once_cell::sync::OnceCell;
|
||||
use serde::de::DeserializeOwned;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::{
|
||||
@@ -639,7 +638,6 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
|
||||
base_internal_url: base_internal_url.to_string(),
|
||||
token,
|
||||
workspace: job.workspace_id.to_string(),
|
||||
client: OnceCell::new(),
|
||||
};
|
||||
|
||||
let last_ping = job.last_ping.clone();
|
||||
@@ -647,6 +645,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
|
||||
db,
|
||||
&client,
|
||||
&job,
|
||||
0,
|
||||
error::Error::ExecutionErr(format!(
|
||||
"Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})",
|
||||
last_ping
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
use windmill_api_client::types::{NewScript, NewScriptLanguage};
|
||||
use std::str::FromStr;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use chrono::Timelike;
|
||||
@@ -7,6 +9,7 @@ use futures::StreamExt;
|
||||
use futures::{stream, Stream};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use sqlx::types::Json;
|
||||
use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
@@ -22,7 +25,6 @@ use sqlx::query;
|
||||
#[cfg(feature = "enterprise")]
|
||||
use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs};
|
||||
|
||||
use windmill_api_client::types::{NewScript, NewScriptLanguage};
|
||||
|
||||
use windmill_common::worker::WORKER_CONFIG;
|
||||
use windmill_common::{
|
||||
@@ -34,7 +36,6 @@ use windmill_common::{
|
||||
use windmill_queue::PushIsolationLevel;
|
||||
use serde::Serialize;
|
||||
|
||||
use std::str::FromStr;
|
||||
|
||||
#[derive(Debug, sqlx::FromRow, Serialize)]
|
||||
pub struct CompletedJob {
|
||||
@@ -872,12 +873,12 @@ impl RunJob {
|
||||
async fn push(self, db: &Pool<Postgres>) -> Uuid {
|
||||
let RunJob { payload, args } = self;
|
||||
let tx = PushIsolationLevel::IsolatedRoot(db.clone(), None);
|
||||
let (uuid, tx) = windmill_queue::push::<rsmq_async::MultiplexedRsmq>(
|
||||
let (uuid, tx) = windmill_queue::push::<_, rsmq_async::MultiplexedRsmq>(
|
||||
&db,
|
||||
tx,
|
||||
"test-workspace",
|
||||
payload,
|
||||
args,
|
||||
Json(args),
|
||||
/* user */ "test-user",
|
||||
/* email */ "test@windmill.dev",
|
||||
/* permissioned_as */ "u/test-user".to_string(),
|
||||
@@ -1874,7 +1875,7 @@ async fn test_invalid_first_step(db: Pool<Postgres>) {
|
||||
|
||||
assert_eq!(
|
||||
job.json_result().unwrap(),
|
||||
serde_json::json!( {"error": {"name": "InternalErr", "message": "Expected an array value, found: {}"}})
|
||||
serde_json::json!( {"error": {"name": "InternalErr", "message": "Expected an array value, found: invalid type: map, expected a sequence at line 1 column 0"}})
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ use axum::{
|
||||
use hyper::StatusCode;
|
||||
use magic_crypt::MagicCryptTrait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Map, Value};
|
||||
use serde_json::{json, value::RawValue};
|
||||
use sha2::{Digest, Sha256};
|
||||
use sql_builder::{bind::Bind, SqlBuilder};
|
||||
use sqlx::{types::Uuid, FromRow};
|
||||
@@ -40,7 +40,7 @@ use windmill_common::{
|
||||
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath,
|
||||
},
|
||||
};
|
||||
use windmill_queue::{push, PushIsolationLevel, QueueTransaction};
|
||||
use windmill_queue::{push, PushArgs, PushIsolationLevel, QueueTransaction};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
@@ -124,7 +124,7 @@ pub struct AppWithLastVersionAndDraft {
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
pub type StaticFields = Map<String, Value>;
|
||||
pub type StaticFields = HashMap<String, Box<RawValue>>;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
@@ -518,7 +518,7 @@ async fn create_app(
|
||||
tx,
|
||||
&w_id,
|
||||
JobPayload::AppDependencies { path: app.path.clone(), version: v_id },
|
||||
serde_json::Map::new(),
|
||||
PushArgs::empty(),
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
windmill_common::users::username_to_permissioned_as(&authed.username),
|
||||
@@ -761,7 +761,7 @@ async fn update_app(
|
||||
tx,
|
||||
&w_id,
|
||||
JobPayload::AppDependencies { path: npath.clone(), version: v_id },
|
||||
serde_json::Map::new(),
|
||||
PushArgs::empty(),
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
windmill_common::users::username_to_permissioned_as(&authed.username),
|
||||
@@ -798,7 +798,7 @@ async fn update_app(
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct ExecuteApp {
|
||||
pub args: Map<String, serde_json::Value>,
|
||||
pub args: Box<RawValue>,
|
||||
// - script: script/<path>
|
||||
// - flow: flow/<path>
|
||||
pub path: Option<String>,
|
||||
@@ -900,17 +900,17 @@ async fn execute_component(
|
||||
}
|
||||
};
|
||||
|
||||
let (job_payload, args, tag) = match &payload {
|
||||
let (job_payload, args, tag) = match payload {
|
||||
ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => {
|
||||
let content = &raw_code.content;
|
||||
let payload = JobPayload::Code(raw_code.clone());
|
||||
let path = digest(content);
|
||||
let args = build_args(policy, component, path, args)?;
|
||||
let args = build_args(policy, &component, path, args)?;
|
||||
(payload, args, None)
|
||||
}
|
||||
ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => {
|
||||
let (payload, tag) = get_payload_tag_from_prefixed_path(path, &db, &w_id).await?;
|
||||
let args = build_args(policy, component, path.to_string(), args)?;
|
||||
let (payload, tag) = get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?;
|
||||
let args = build_args(policy, &component, path.to_string(), args)?;
|
||||
(payload, args, tag)
|
||||
}
|
||||
_ => unreachable!(),
|
||||
@@ -1002,11 +1002,17 @@ fn build_args(
|
||||
policy: Policy,
|
||||
component: &str,
|
||||
path: String,
|
||||
args: &Map<String, Value>,
|
||||
) -> Result<Map<String, Value>> {
|
||||
args: Box<RawValue>,
|
||||
) -> Result<PushArgs<Box<RawValue>>> {
|
||||
// disallow var and res access in args coming from the user for security reasons
|
||||
args.into_iter()
|
||||
.try_for_each(|x| disallow_var_res_access(x.1))?;
|
||||
{
|
||||
let args_str = args.to_string();
|
||||
if args_str.contains("$var:") || args_str.contains("$res:") {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"For security reasons, variable or resource access is not allowed as dynamic argument"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let key = format!("{}:{}", component, &path);
|
||||
let static_args = policy
|
||||
.triggerables
|
||||
@@ -1015,7 +1021,7 @@ fn build_args(
|
||||
.map(|x| x.clone())
|
||||
.or_else(|| {
|
||||
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
|
||||
Some(Map::new())
|
||||
Some(HashMap::new())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -1023,26 +1029,9 @@ fn build_args(
|
||||
.ok_or_else(|| {
|
||||
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
|
||||
})?;
|
||||
let mut args = args.clone();
|
||||
let mut extra = HashMap::new();
|
||||
for (k, v) in static_args {
|
||||
args.insert(k.to_string(), v.to_owned());
|
||||
}
|
||||
Ok(args)
|
||||
}
|
||||
|
||||
fn disallow_var_res_access(args: &serde_json::Value) -> Result<()> {
|
||||
match args {
|
||||
Value::Object(v) => v.into_iter().try_for_each(|x| disallow_var_res_access(x.1)),
|
||||
Value::Array(arr) => arr.into_iter().try_for_each(|v| disallow_var_res_access(v)),
|
||||
Value::String(s) => {
|
||||
if s.starts_with("$var:") || s.starts_with("$res:") {
|
||||
Err(Error::BadRequest(format!(
|
||||
"For security reasons, variable or resource access is not allowed as dynamic argument"
|
||||
)))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
_ => Ok(()),
|
||||
extra.insert(k.to_string(), v.to_owned());
|
||||
}
|
||||
Ok(PushArgs { extra, args: sqlx::types::Json(args) })
|
||||
}
|
||||
|
||||
@@ -6,23 +6,24 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use axum::{
|
||||
extract::{Extension, Path, Query},
|
||||
extract::{Extension, Path},
|
||||
routing::{get, post, put},
|
||||
Router,
|
||||
};
|
||||
use hyper::{HeaderMap, StatusCode};
|
||||
use serde::Deserialize;
|
||||
use hyper::StatusCode;
|
||||
use serde_json::value::RawValue;
|
||||
use sqlx::types::Json;
|
||||
use windmill_common::{
|
||||
db::UserDB,
|
||||
error::{JsonResult, Result},
|
||||
utils::{not_found_if_none, StripPath},
|
||||
};
|
||||
use windmill_queue::PushArgs;
|
||||
|
||||
use crate::{
|
||||
db::{ApiAuthed, DB},
|
||||
jobs::{add_include_headers, add_raw_string, JsonOrForm},
|
||||
};
|
||||
use crate::db::{ApiAuthed, DB};
|
||||
|
||||
const KEEP_LAST: i64 = 8;
|
||||
|
||||
@@ -85,21 +86,12 @@ pub async fn new_payload(
|
||||
Ok(StatusCode::CREATED)
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone)]
|
||||
pub struct IncludeHeaderQuery {
|
||||
include_header: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn update_payload(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<IncludeHeaderQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args, raw_string): JsonOrForm,
|
||||
args: PushArgs<HashMap<String, Box<RawValue>>>,
|
||||
) -> Result<StatusCode> {
|
||||
let mut tx = db.begin().await?;
|
||||
let args = add_include_headers(&run_query.include_header, headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
|
||||
sqlx::query!(
|
||||
"
|
||||
@@ -110,7 +102,7 @@ pub async fn update_payload(
|
||||
",
|
||||
&w_id,
|
||||
&path.to_path(),
|
||||
serde_json::json!(args),
|
||||
Json(args) as Json<PushArgs<HashMap<String, Box<RawValue>>>>,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
@@ -38,6 +38,7 @@ use windmill_common::{
|
||||
scripts::Schema,
|
||||
utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, StripPath},
|
||||
};
|
||||
use windmill_queue::PushArgs;
|
||||
use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
@@ -290,7 +291,7 @@ async fn create_flow(
|
||||
tx,
|
||||
&w_id,
|
||||
JobPayload::FlowDependencies { path: nf.path.clone() },
|
||||
serde_json::Map::new(),
|
||||
PushArgs::empty(),
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
windmill_common::users::username_to_permissioned_as(&authed.username),
|
||||
@@ -482,7 +483,7 @@ async fn update_flow(
|
||||
tx,
|
||||
&w_id,
|
||||
JobPayload::FlowDependencies { path: nf.path.clone() },
|
||||
serde_json::Map::new(),
|
||||
PushArgs::empty(),
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
windmill_common::users::username_to_permissioned_as(&authed.username),
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::db::ApiAuthed;
|
||||
|
||||
use crate::{
|
||||
@@ -19,13 +21,12 @@ use axum::{
|
||||
extract::{FromRequest, Json, Path, Query},
|
||||
response::{IntoResponse, Response},
|
||||
routing::{get, post},
|
||||
Extension, Form, RequestExt, Router,
|
||||
Extension, Router,
|
||||
};
|
||||
use base64::Engine;
|
||||
use bytes::Bytes;
|
||||
use chrono::Utc;
|
||||
use hmac::Mac;
|
||||
use hyper::{header::CONTENT_TYPE, http, HeaderMap, Request, StatusCode};
|
||||
use hyper::{http, Request, StatusCode};
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use sql_builder::{prelude::*, quote, SqlBuilder};
|
||||
use sqlx::types::JsonRawValue;
|
||||
@@ -46,7 +47,7 @@ use windmill_common::{
|
||||
users::username_to_permissioned_as,
|
||||
utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
|
||||
};
|
||||
use windmill_queue::{job_is_complete, push, PushIsolationLevel};
|
||||
use windmill_queue::{empty_args, job_is_complete, push, PushArgs, PushIsolationLevel};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
let cors = CorsLayer::new()
|
||||
@@ -87,18 +88,6 @@ pub fn workspaced_service() -> Router {
|
||||
.head(|| async { "" })
|
||||
.layer(cors.clone()),
|
||||
)
|
||||
.route(
|
||||
"/openai_sync/p/*script_path",
|
||||
post(openai_sync_script_by_path)
|
||||
.head(|| async { "" })
|
||||
.layer(cors.clone()),
|
||||
)
|
||||
.route(
|
||||
"/openai_sync/f/*script_path",
|
||||
post(openai_sync_flow_by_path)
|
||||
.head(|| async { "" })
|
||||
.layer(cors.clone()),
|
||||
)
|
||||
.route(
|
||||
"/run/h/:hash",
|
||||
post(run_job_by_hash)
|
||||
@@ -190,11 +179,16 @@ pub fn global_root_service() -> Router {
|
||||
Router::new().route("/db_clock", get(get_db_clock))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct JsonPath {
|
||||
pub json_path: Option<String>,
|
||||
}
|
||||
async fn get_result_by_id(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, flow_id, node_id)): Path<(String, Uuid, String)>,
|
||||
) -> windmill_common::error::JsonResult<serde_json::Value> {
|
||||
let res = windmill_queue::get_result_by_id(db, w_id, flow_id, node_id).await?;
|
||||
Query(JsonPath { json_path }): Query<JsonPath>,
|
||||
) -> windmill_common::error::JsonResult<Box<JsonRawValue>> {
|
||||
let res = windmill_queue::get_result_by_id(db, w_id, flow_id, node_id, json_path).await?;
|
||||
Ok(Json(res))
|
||||
}
|
||||
|
||||
@@ -350,7 +344,7 @@ async fn get_job(
|
||||
) -> error::Result<Response> {
|
||||
let cjob_option = sqlx::query("SELECT
|
||||
id, workspace_id, parent_job, created_by, created_at, duration_ms, success, script_hash, script_path,
|
||||
args, CASE WHEN pg_column_size(result) < 2000000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind, env_id,
|
||||
CASE WHEN pg_column_size(args) < 2000000 THEN args ELSE '{\"reason\": \"WINDMILL_TOO_BIG\"}'::jsonb END as args, CASE WHEN pg_column_size(result) < 2000000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind, env_id,
|
||||
schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language, started_at, is_skipped,
|
||||
raw_lock, email, visible_to_owner, mem_peak, tag
|
||||
FROM completed_job WHERE id = $1 AND workspace_id = $2")
|
||||
@@ -363,7 +357,11 @@ async fn get_job(
|
||||
Ok(Json(job).into_response())
|
||||
} else {
|
||||
let job_o = sqlx::query_as::<_, QueuedJob>(
|
||||
"SELECT *
|
||||
"SELECT id, workspace_id, parent_job, created_by, created_at, started_at, scheduled_for, running,
|
||||
script_hash, script_path, CASE WHEN pg_column_size(args) < 2000000 THEN args ELSE '{\"reason\": \"WINDMILL_TOO_BIG\"}'::jsonb END as args, logs, raw_code, canceled, canceled_by, canceled_reason, last_ping,
|
||||
job_kind, env_id, schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language,
|
||||
suspend, suspend_until, same_worker, raw_lock, pre_run_error, email, visible_to_owner, mem_peak,
|
||||
root_job, leaf_jobs, tag, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, cache_ttl
|
||||
FROM queue WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(id)
|
||||
@@ -407,7 +405,7 @@ pub struct CompletedJob<'rows> {
|
||||
pub script_hash: Option<ScriptHash>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_path: Option<String>,
|
||||
pub args: Option<serde_json::Value>,
|
||||
pub args: Option<&'rows JsonRawValue>,
|
||||
#[serde(skip_serializing_if = "Option::is_none", borrow)]
|
||||
pub result: Option<&'rows JsonRawValue>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
@@ -502,21 +500,12 @@ pub struct RunJobQuery {
|
||||
scheduled_for: Option<chrono::DateTime<chrono::Utc>>,
|
||||
scheduled_in_secs: Option<i64>,
|
||||
parent_job: Option<Uuid>,
|
||||
include_header: Option<String>,
|
||||
invisible_to_owner: Option<bool>,
|
||||
queue_limit: Option<i64>,
|
||||
payload: Option<String>,
|
||||
job_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref INCLUDE_HEADERS: Vec<String> = std::env::var("INCLUDE_HEADERS")
|
||||
.ok().map(|x| x
|
||||
.split(',')
|
||||
.map(|s| s.to_string())
|
||||
.collect()).unwrap_or_default();
|
||||
}
|
||||
|
||||
impl RunJobQuery {
|
||||
async fn get_scheduled_for<'c>(
|
||||
&self,
|
||||
@@ -531,38 +520,6 @@ impl RunJobQuery {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
fn add_include_headers(
|
||||
&self,
|
||||
headers: HeaderMap,
|
||||
args: serde_json::Map<String, serde_json::Value>,
|
||||
) -> serde_json::Map<String, serde_json::Value> {
|
||||
return add_include_headers(&self.include_header, headers, args);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_include_headers(
|
||||
include_header: &Option<String>,
|
||||
headers: HeaderMap,
|
||||
mut args: serde_json::Map<String, serde_json::Value>,
|
||||
) -> serde_json::Map<String, serde_json::Value> {
|
||||
let whitelist = include_header
|
||||
.as_ref()
|
||||
.map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
|
||||
.unwrap_or_default();
|
||||
|
||||
whitelist
|
||||
.iter()
|
||||
.chain(INCLUDE_HEADERS.iter())
|
||||
.for_each(|h| {
|
||||
if let Some(v) = headers.get(h) {
|
||||
args.insert(
|
||||
h.to_string().to_lowercase().replace('-', "_"),
|
||||
serde_json::Value::String(v.to_str().unwrap().to_string()),
|
||||
);
|
||||
}
|
||||
});
|
||||
args
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -1485,7 +1442,7 @@ struct Preview {
|
||||
content: Option<String>,
|
||||
kind: Option<PreviewKind>,
|
||||
path: Option<String>,
|
||||
args: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
args: Option<Box<JsonRawValue>>,
|
||||
language: Option<ScriptLang>,
|
||||
tag: Option<String>,
|
||||
}
|
||||
@@ -1494,78 +1451,10 @@ struct Preview {
|
||||
struct PreviewFlow {
|
||||
value: FlowValue,
|
||||
path: Option<String>,
|
||||
args: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
args: Option<Box<JsonRawValue>>,
|
||||
tag: Option<String>,
|
||||
}
|
||||
|
||||
pub struct JsonOrForm(
|
||||
pub Option<serde_json::Map<String, serde_json::Value>>,
|
||||
pub Option<String>,
|
||||
);
|
||||
|
||||
#[axum::async_trait]
|
||||
impl<S> FromRequest<S, axum::body::Body> for JsonOrForm
|
||||
where
|
||||
S: Send + Sync,
|
||||
{
|
||||
type Rejection = Response;
|
||||
|
||||
async fn from_request(
|
||||
req: Request<axum::body::Body>,
|
||||
_state: &S,
|
||||
) -> Result<Self, Self::Rejection> {
|
||||
let content_type_header = req.headers().get(CONTENT_TYPE);
|
||||
let content_type = content_type_header.and_then(|value| value.to_str().ok());
|
||||
if content_type.is_none() || content_type.unwrap().starts_with("application/json") {
|
||||
if req
|
||||
.uri()
|
||||
.query()
|
||||
.map(|x| x.contains("raw=true"))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
let bytes = Bytes::from_request(req, _state)
|
||||
.await
|
||||
.map_err(IntoResponse::into_response)?;
|
||||
let str = String::from_utf8(bytes.to_vec()).map_err(|e| {
|
||||
Error::BadRequest(format!("invalid utf8: {}", e)).into_response()
|
||||
})?;
|
||||
let payload =
|
||||
serde_json::from_str::<Option<serde_json::Value>>(&str).map_err(|e| {
|
||||
Error::BadRequest(format!("invalid json: {}", e)).into_response()
|
||||
})?;
|
||||
return match payload {
|
||||
Some(serde_json::Value::Object(map)) => Ok(Self(Some(map), Some(str))),
|
||||
None => Ok(Self(None, Some(str))),
|
||||
Some(x) => {
|
||||
let mut map = serde_json::Map::new();
|
||||
map.insert("body".to_string(), x);
|
||||
Ok(Self(Some(map), Some(str)))
|
||||
}
|
||||
};
|
||||
} else {
|
||||
let Json(payload): Json<Option<serde_json::Value>> =
|
||||
req.extract().await.map_err(IntoResponse::into_response)?;
|
||||
return match payload {
|
||||
Some(serde_json::Value::Object(map)) => Ok(Self(Some(map), None)),
|
||||
None => Ok(Self(None, None)),
|
||||
Some(x) => {
|
||||
let mut map = serde_json::Map::new();
|
||||
map.insert("body".to_string(), x);
|
||||
Ok(Self(Some(map), None))
|
||||
}
|
||||
};
|
||||
}
|
||||
} else if content_type
|
||||
.unwrap()
|
||||
.starts_with("application/x-www-form-urlencoded")
|
||||
{
|
||||
let Form(payload) = req.extract().await.map_err(IntoResponse::into_response)?;
|
||||
return Ok(Self(Some(payload), None));
|
||||
} else {
|
||||
Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response())
|
||||
}
|
||||
}
|
||||
}
|
||||
pub struct QueryOrBody<D>(pub Option<D>);
|
||||
|
||||
#[axum::async_trait]
|
||||
@@ -1673,8 +1562,7 @@ pub async fn run_flow_by_path(
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path((w_id, flow_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args, raw_string): JsonOrForm,
|
||||
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
check_license_key_valid().await?;
|
||||
@@ -1691,8 +1579,6 @@ pub async fn run_flow_by_path(
|
||||
.flatten();
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
let (uuid, tx) = push(
|
||||
&db,
|
||||
@@ -1728,8 +1614,7 @@ pub async fn run_job_by_path(
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path((w_id, script_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args, raw_string): JsonOrForm,
|
||||
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
check_license_key_valid().await?;
|
||||
@@ -1740,8 +1625,6 @@ pub async fn run_job_by_path(
|
||||
|
||||
let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
@@ -1934,12 +1817,14 @@ pub async fn run_wait_result_job_by_path_get(
|
||||
.map(decode_payload)
|
||||
.map(|x| x.map_err(|e| Error::InternalErr(e.to_string())));
|
||||
|
||||
let args = if let Some(payload) = payload_r {
|
||||
let payload_args = if let Some(payload) = payload_r {
|
||||
payload?
|
||||
} else {
|
||||
serde_json::Map::new()
|
||||
HashMap::new()
|
||||
};
|
||||
|
||||
let args = PushArgs { extra: payload_args, args: sqlx::types::Json(empty_args()) };
|
||||
|
||||
check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
|
||||
let script_path = script_path.to_path();
|
||||
check_scopes(&authed, || format!("run:script/{script_path}"))?;
|
||||
@@ -1983,7 +1868,7 @@ pub async fn run_wait_result_flow_by_path_get(
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, flow_path)): Path<(String, StripPath)>,
|
||||
headers: HeaderMap,
|
||||
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
@@ -1998,22 +1883,16 @@ pub async fn run_wait_result_flow_by_path_get(
|
||||
.map(decode_payload)
|
||||
.map(|x| x.map_err(|e| Error::InternalErr(e.to_string())));
|
||||
|
||||
let args = if let Some(payload) = payload_r {
|
||||
let payload_args = if let Some(payload) = payload_r {
|
||||
payload?
|
||||
} else {
|
||||
serde_json::Map::new()
|
||||
HashMap::new()
|
||||
};
|
||||
|
||||
let args = PushArgs { extra: payload_args, args: sqlx::types::Json(HashMap::new()) };
|
||||
|
||||
run_wait_result_flow_by_path_internal(
|
||||
db,
|
||||
run_query,
|
||||
flow_path,
|
||||
authed,
|
||||
rsmq,
|
||||
user_db,
|
||||
headers,
|
||||
Some(args),
|
||||
None,
|
||||
w_id,
|
||||
db, run_query, flow_path, authed, rsmq, user_db, args, w_id,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2025,8 +1904,7 @@ pub async fn run_wait_result_script_by_path(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, script_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args, raw_string): JsonOrForm,
|
||||
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
check_license_key_valid().await?;
|
||||
@@ -2039,58 +1917,7 @@ pub async fn run_wait_result_script_by_path(
|
||||
rsmq,
|
||||
user_db,
|
||||
w_id,
|
||||
headers,
|
||||
args,
|
||||
raw_string,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn convert_from_openai_json(
|
||||
json: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
) -> error::Result<Option<serde_json::Map<String, serde_json::Value>>> {
|
||||
if let Some(m) = json {
|
||||
let mut new_json = serde_json::Map::new();
|
||||
let input_keys = m
|
||||
.get("inputKeys")
|
||||
.and_then(|x| x.as_array())
|
||||
.map(|x| x.to_owned())
|
||||
.unwrap_or_default();
|
||||
let input_values = m
|
||||
.get("inputValues")
|
||||
.and_then(|x| x.as_array())
|
||||
.map(|x| x.to_owned())
|
||||
.unwrap_or_default();
|
||||
for (k, v) in input_keys.into_iter().zip(input_values.into_iter()) {
|
||||
new_json.insert(k.as_str().unwrap_or_else(|| "invalid_key").to_string(), v);
|
||||
}
|
||||
Ok(Some(new_json))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn openai_sync_script_by_path(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, script_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args, raw_string): JsonOrForm,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
run_wait_result_script_by_path_internal(
|
||||
db,
|
||||
run_query,
|
||||
script_path,
|
||||
authed,
|
||||
rsmq,
|
||||
user_db,
|
||||
w_id,
|
||||
headers,
|
||||
convert_from_openai_json(args)?,
|
||||
raw_string,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2103,9 +1930,7 @@ async fn run_wait_result_script_by_path_internal(
|
||||
rsmq: Option<rsmq_async::MultiplexedRsmq>,
|
||||
user_db: UserDB,
|
||||
w_id: String,
|
||||
headers: HeaderMap,
|
||||
args: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
raw_string: Option<String>,
|
||||
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
|
||||
) -> Result<Json<serde_json::Value>, Error> {
|
||||
check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
|
||||
let script_path = script_path.to_path();
|
||||
@@ -2113,8 +1938,6 @@ async fn run_wait_result_script_by_path_internal(
|
||||
|
||||
let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?;
|
||||
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
@@ -2153,8 +1976,7 @@ pub async fn run_wait_result_script_by_hash(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args, raw_string): JsonOrForm,
|
||||
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
check_license_key_valid().await?;
|
||||
@@ -2173,8 +1995,6 @@ pub async fn run_wait_result_script_by_hash(
|
||||
) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?;
|
||||
check_scopes(&authed, || format!("run:script/{path}"))?;
|
||||
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
@@ -2214,31 +2034,6 @@ pub async fn run_wait_result_script_by_hash(
|
||||
run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_hash))).await
|
||||
}
|
||||
|
||||
pub async fn openai_sync_flow_by_path(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, flow_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args, raw_string): JsonOrForm,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
run_wait_result_flow_by_path_internal(
|
||||
db,
|
||||
run_query,
|
||||
flow_path,
|
||||
authed,
|
||||
rsmq,
|
||||
user_db,
|
||||
headers,
|
||||
convert_from_openai_json(args)?,
|
||||
raw_string,
|
||||
w_id,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn run_wait_result_flow_by_path(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -2246,14 +2041,13 @@ pub async fn run_wait_result_flow_by_path(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, flow_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args, raw_string): JsonOrForm,
|
||||
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
check_license_key_valid().await?;
|
||||
|
||||
run_wait_result_flow_by_path_internal(
|
||||
db, run_query, flow_path, authed, rsmq, user_db, headers, args, raw_string, w_id,
|
||||
db, run_query, flow_path, authed, rsmq, user_db, args, w_id,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2265,9 +2059,7 @@ async fn run_wait_result_flow_by_path_internal(
|
||||
authed: ApiAuthed,
|
||||
rsmq: Option<rsmq_async::MultiplexedRsmq>,
|
||||
user_db: UserDB,
|
||||
headers: HeaderMap,
|
||||
args: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
raw_string: Option<String>,
|
||||
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
|
||||
w_id: String,
|
||||
) -> Result<Json<serde_json::Value>, Error> {
|
||||
check_queue_too_long(&db, run_query.queue_limit).await?;
|
||||
@@ -2276,8 +2068,7 @@ async fn run_wait_result_flow_by_path_internal(
|
||||
check_scopes(&authed, || format!("run:flow/{flow_path}"))?;
|
||||
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
|
||||
let tag = sqlx::query_scalar!(
|
||||
"SELECT tag from flow WHERE path = $1 and workspace_id = $2",
|
||||
flow_path,
|
||||
@@ -2324,7 +2115,6 @@ async fn run_preview_job(
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
Json(preview): Json<Preview>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
@@ -2337,7 +2127,6 @@ async fn run_preview_job(
|
||||
));
|
||||
}
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let args = run_query.add_include_headers(headers, preview.args.unwrap_or_default());
|
||||
check_tag_available_for_workspace(&w_id, &preview.tag).await?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
@@ -2358,7 +2147,7 @@ async fn run_preview_job(
|
||||
cache_ttl: None,
|
||||
}),
|
||||
},
|
||||
args,
|
||||
preview.args.unwrap_or_default(),
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
@@ -2436,7 +2225,7 @@ async fn add_batch_jobs(
|
||||
value: batch_info.flow_value.clone().unwrap(),
|
||||
path: None,
|
||||
},
|
||||
serde_json::Map::new(),
|
||||
PushArgs::empty(),
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
@@ -2519,7 +2308,6 @@ async fn run_preview_flow_job(
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
Json(raw_flow): Json<PreviewFlow>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
check_scopes(&authed, || format!("runflow"))?;
|
||||
@@ -2529,7 +2317,6 @@ async fn run_preview_flow_job(
|
||||
));
|
||||
}
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let args = run_query.add_include_headers(headers, raw_flow.args.unwrap_or_default());
|
||||
check_tag_available_for_workspace(&w_id, &raw_flow.tag).await?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
@@ -2538,7 +2325,7 @@ async fn run_preview_flow_job(
|
||||
tx,
|
||||
&w_id,
|
||||
JobPayload::RawFlow { value: raw_flow.value, path: raw_flow.path },
|
||||
args,
|
||||
raw_flow.args.unwrap_or_default(),
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
@@ -2569,8 +2356,7 @@ pub async fn run_job_by_hash(
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args, raw_string): JsonOrForm,
|
||||
args: PushArgs<HashMap<String, Box<JsonRawValue>>>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
check_license_key_valid().await?;
|
||||
@@ -2588,8 +2374,7 @@ pub async fn run_job_by_hash(
|
||||
check_scopes(&authed, || format!("run:script/{path}"))?;
|
||||
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
@@ -2648,21 +2433,22 @@ async fn get_job_update(
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
Query(JobUpdateQuery { running, log_offset }): Query<JobUpdateQuery>,
|
||||
) -> error::JsonResult<JobUpdate> {
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
let record = sqlx::query!(
|
||||
"SELECT substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3",
|
||||
"SELECT running, substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3",
|
||||
log_offset,
|
||||
&w_id,
|
||||
&id
|
||||
)
|
||||
.fetch_optional(&mut *tx)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
|
||||
if let Some(record) = record {
|
||||
tx.commit().await?;
|
||||
Ok(Json(JobUpdate {
|
||||
running: if !running { Some(true) } else { None },
|
||||
running: if !running && record.running {
|
||||
Some(true)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
completed: None,
|
||||
new_logs: record.logs,
|
||||
mem_peak: record.mem_peak,
|
||||
@@ -2675,10 +2461,9 @@ async fn get_job_update(
|
||||
&w_id,
|
||||
&id
|
||||
)
|
||||
.fetch_optional(&mut *tx)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
let logs = not_found_if_none(logs, "Job Update", id.to_string())?;
|
||||
tx.commit().await?;
|
||||
Ok(Json(JobUpdate {
|
||||
running: Some(false),
|
||||
completed: Some(true),
|
||||
@@ -2851,7 +2636,10 @@ async fn get_completed_job<'a>(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
) -> error::Result<Response> {
|
||||
let job_o = sqlx::query("SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2")
|
||||
let job_o = sqlx::query("SELECT id, workspace_id, parent_job, created_by, created_at, duration_ms, success, script_hash, script_path,
|
||||
CASE WHEN pg_column_size(args) < 2000000 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args, CASE WHEN pg_column_size(result) < 2000000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind, env_id,
|
||||
schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language, started_at, is_skipped,
|
||||
raw_lock, email, visible_to_owner, mem_peak, tag FROM completed_job WHERE id = $1 AND workspace_id = $2")
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&db)
|
||||
@@ -2875,13 +2663,29 @@ impl<'a> IntoResponse for RawResult<'a> {
|
||||
async fn get_completed_job_result(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
Query(JsonPath { json_path }): Query<JsonPath>,
|
||||
) -> error::Result<Response> {
|
||||
let result_o =
|
||||
let result_o = if let Some(json_path) = json_path {
|
||||
sqlx::query(
|
||||
"SELECT result #> $3 as result FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.bind(
|
||||
json_path
|
||||
.split(".")
|
||||
.map(|x| x.to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?
|
||||
} else {
|
||||
sqlx::query("SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2")
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
.await?
|
||||
};
|
||||
|
||||
let result = not_found_if_none(result_o, "Completed Job", id.to_string())?;
|
||||
Ok(RawResult::from_row(&result)?.into_response())
|
||||
|
||||
@@ -201,7 +201,6 @@ pub async fn run_server(
|
||||
users::global_service().layer(Extension(argon2.clone())),
|
||||
)
|
||||
.nest("/settings", settings::global_service())
|
||||
.nest("/jobs", jobs::global_root_service())
|
||||
.nest("/workers", workers::global_service())
|
||||
.nest("/configs", configs::global_service())
|
||||
.nest("/scripts", scripts::global_service())
|
||||
@@ -212,6 +211,7 @@ pub async fn run_server(
|
||||
.nest("/schedules", schedule::global_service())
|
||||
.route_layer(from_extractor::<ApiAuthed>())
|
||||
.route_layer(from_extractor::<users::Tokened>())
|
||||
.nest("/jobs", jobs::global_root_service())
|
||||
.nest(
|
||||
"/saml",
|
||||
saml::global_service().layer(Extension(Arc::new(sp_extension.0))),
|
||||
|
||||
@@ -903,7 +903,7 @@ async fn slack_command(
|
||||
tx,
|
||||
&settings.workspace_id,
|
||||
payload,
|
||||
map,
|
||||
sqlx::types::Json(map),
|
||||
&form.user_name,
|
||||
&settings.slack_email,
|
||||
"g/slack".to_string(),
|
||||
|
||||
@@ -21,7 +21,7 @@ use axum::{
|
||||
};
|
||||
use hyper::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use serde_json::{value::RawValue, Value};
|
||||
use sql_builder::{bind::Bind, SqlBuilder};
|
||||
use sqlx::{FromRow, Postgres, Transaction};
|
||||
use uuid::Uuid;
|
||||
@@ -117,7 +117,7 @@ pub struct ListableResource {
|
||||
#[derive(Deserialize)]
|
||||
pub struct CreateResource {
|
||||
pub path: String,
|
||||
pub value: Option<serde_json::Value>,
|
||||
pub value: Option<Box<RawValue>>,
|
||||
pub description: Option<String>,
|
||||
pub resource_type: String,
|
||||
}
|
||||
@@ -125,7 +125,7 @@ pub struct CreateResource {
|
||||
struct EditResource {
|
||||
path: Option<String>,
|
||||
description: Option<String>,
|
||||
value: Option<serde_json::Value>,
|
||||
value: Option<Box<RawValue>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -496,6 +496,9 @@ async fn create_resource(
|
||||
check_path_conflict(&mut tx, &w_id, &resource.path).await?;
|
||||
}
|
||||
|
||||
let res_value = resource.value.unwrap_or_default();
|
||||
let raw_json = sqlx::types::Json(res_value.as_ref());
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO resource
|
||||
(workspace_id, path, value, description, resource_type)
|
||||
@@ -503,7 +506,7 @@ async fn create_resource(
|
||||
DO UPDATE SET value = $3, description = $4, resource_type = $5",
|
||||
w_id,
|
||||
resource.path,
|
||||
resource.value,
|
||||
raw_json as sqlx::types::Json<&RawValue>,
|
||||
resource.description,
|
||||
resource.resource_type,
|
||||
)
|
||||
|
||||
@@ -46,7 +46,9 @@ use windmill_common::{
|
||||
not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
|
||||
},
|
||||
};
|
||||
use windmill_queue::{self, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
|
||||
use windmill_queue::{
|
||||
self, schedule::push_scheduled_job, PushArgs, PushIsolationLevel, QueueTransaction,
|
||||
};
|
||||
|
||||
const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
|
||||
|
||||
@@ -592,7 +594,7 @@ async fn create_script(
|
||||
tx,
|
||||
&w_id,
|
||||
JobPayload::Dependencies { hash, dependencies, language: ns.language, path: ns.path },
|
||||
serde_json::Map::new(),
|
||||
PushArgs::empty(),
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
|
||||
@@ -549,7 +549,7 @@ async fn run_slack_message_test_job(
|
||||
false,
|
||||
w_id.as_str(),
|
||||
&format!("script/{}", req.hub_script_path.as_str()),
|
||||
&json!(fake_result),
|
||||
sqlx::types::Json(&fake_result),
|
||||
0,
|
||||
Utc::now(),
|
||||
Some(json!(extra_args)),
|
||||
|
||||
@@ -11,8 +11,8 @@ use std::time::Duration;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::flows::FlowValue;
|
||||
use crate::more_serde::default_false;
|
||||
use crate::{flows::FlowValue, more_serde::is_default};
|
||||
|
||||
const MINUTES: Duration = Duration::from_secs(60);
|
||||
const HOURS: Duration = MINUTES.saturating_mul(60);
|
||||
@@ -20,21 +20,24 @@ const HOURS: Duration = MINUTES.saturating_mul(60);
|
||||
pub const MAX_RETRY_ATTEMPTS: u16 = 1000;
|
||||
pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6);
|
||||
|
||||
pub fn is_retry_default(v: &RetryStatus) -> bool {
|
||||
v.fail_count == 0 && v.failed_jobs.is_empty()
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct FlowStatus {
|
||||
pub step: i32,
|
||||
pub modules: Vec<FlowStatusModule>,
|
||||
pub failure_module: FlowStatusModuleWParent,
|
||||
#[serde(default)]
|
||||
#[serde(skip_serializing_if = "is_default")]
|
||||
#[serde(skip_serializing_if = "is_retry_default")]
|
||||
pub retry: RetryStatus,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||
#[serde(default)]
|
||||
pub struct RetryStatus {
|
||||
pub fail_count: u16,
|
||||
pub previous_result: Option<serde_json::Value>,
|
||||
pub failed_jobs: Vec<Uuid>,
|
||||
}
|
||||
|
||||
@@ -47,7 +50,6 @@ pub struct Iterator {
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct BranchAllStatus {
|
||||
pub branch: usize,
|
||||
pub previous_result: serde_json::Value,
|
||||
pub len: usize,
|
||||
}
|
||||
|
||||
@@ -189,7 +191,7 @@ impl FlowStatus {
|
||||
.unwrap_or_else(|| "failure".to_string()),
|
||||
},
|
||||
},
|
||||
retry: RetryStatus { fail_count: 0, previous_result: None, failed_jobs: vec![] },
|
||||
retry: RetryStatus { fail_count: 0, failed_jobs: vec![] },
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ use std::{
|
||||
};
|
||||
|
||||
use serde::{self, Deserialize, Serialize, Serializer};
|
||||
use serde_json::value::RawValue;
|
||||
|
||||
use crate::{
|
||||
more_serde::{
|
||||
@@ -171,7 +172,7 @@ pub struct Suspend {
|
||||
pub struct Mock {
|
||||
pub enabled: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub return_value: Option<serde_json::Value>,
|
||||
pub return_value: Option<Box<RawValue>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Debug, Clone)]
|
||||
@@ -203,7 +204,7 @@ impl FlowModule {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
|
||||
#[derive(Deserialize, Serialize, Debug, Clone)]
|
||||
#[serde(
|
||||
tag = "type",
|
||||
rename_all(serialize = "lowercase", deserialize = "lowercase")
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{Pool, Postgres, Transaction};
|
||||
use serde_json::value::RawValue;
|
||||
use sqlx::{types::Json, Pool, Postgres, Transaction};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
@@ -27,7 +30,7 @@ pub enum JobKind {
|
||||
Noop,
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
|
||||
#[derive(sqlx::FromRow, Debug, Serialize, Clone)]
|
||||
pub struct QueuedJob {
|
||||
pub workspace_id: String,
|
||||
pub id: Uuid,
|
||||
@@ -43,7 +46,7 @@ pub struct QueuedJob {
|
||||
pub script_hash: Option<ScriptHash>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_path: Option<String>,
|
||||
pub args: Option<serde_json::Value>,
|
||||
pub args: Option<Json<HashMap<String, Box<RawValue>>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logs: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
@@ -95,6 +98,14 @@ pub struct QueuedJob {
|
||||
}
|
||||
|
||||
impl QueuedJob {
|
||||
pub fn get_args(&self) -> HashMap<String, Box<RawValue>> {
|
||||
if let Some(args) = self.args.as_ref() {
|
||||
args.0.clone()
|
||||
} else {
|
||||
HashMap::new()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn script_path(&self) -> &str {
|
||||
self.script_path
|
||||
.as_ref()
|
||||
@@ -112,9 +123,7 @@ impl QueuedJob {
|
||||
self.script_path()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl QueuedJob {
|
||||
pub fn parse_raw_flow(&self) -> Option<FlowValue> {
|
||||
self.raw_flow
|
||||
.as_ref()
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::{collections::HashMap, sync::Arc};
|
||||
use itertools::Itertools;
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::value::RawValue;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::ServerConfig, DB};
|
||||
@@ -260,3 +261,13 @@ pub struct WorkerConfig {
|
||||
pub additional_python_paths: Option<Vec<String>>,
|
||||
pub pip_local_dependencies: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
pub fn to_raw_value<T: Serialize>(result: &T) -> Box<RawValue> {
|
||||
serde_json::value::to_raw_value(result)
|
||||
.unwrap_or_else(|_| RawValue::from_string("{}".to_string()).unwrap())
|
||||
}
|
||||
|
||||
pub fn to_raw_value_owned(result: serde_json::Value) -> Box<RawValue> {
|
||||
serde_json::value::to_raw_value(&result)
|
||||
.unwrap_or_else(|_| RawValue::from_string("{}".to_string()).unwrap())
|
||||
}
|
||||
|
||||
@@ -37,4 +37,5 @@ tokio.workspace = true
|
||||
futures-core.workspace = true
|
||||
itertools.workspace = true
|
||||
async-recursion.workspace = true
|
||||
bigdecimal.workspace = true
|
||||
bigdecimal.workspace = true
|
||||
axum.workspace = true
|
||||
@@ -10,12 +10,23 @@ use std::{collections::HashMap, vec};
|
||||
|
||||
use anyhow::Context;
|
||||
use async_recursion::async_recursion;
|
||||
use axum::{
|
||||
body::Bytes,
|
||||
extract::FromRequest,
|
||||
http::Request,
|
||||
response::{IntoResponse, Response},
|
||||
Form, RequestExt,
|
||||
};
|
||||
use bigdecimal::ToPrimitive;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use reqwest::Client;
|
||||
use reqwest::{
|
||||
header::{HeaderMap, CONTENT_TYPE},
|
||||
Client, StatusCode,
|
||||
};
|
||||
use rsmq_async::RsmqConnection;
|
||||
use serde_json::json;
|
||||
use sqlx::{Pool, Postgres, Transaction};
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, value::RawValue};
|
||||
use sqlx::{types::Json, FromRow, Pool, Postgres, Transaction};
|
||||
#[cfg(feature = "benchmark")]
|
||||
use std::time::Instant;
|
||||
use tracing::{instrument, Instrument};
|
||||
@@ -36,7 +47,7 @@ use windmill_common::{
|
||||
schedule::{schedule_to_user, Schedule},
|
||||
scripts::{ScriptHash, ScriptLang},
|
||||
users::{username_to_permissioned_as, SUPERADMIN_SECRET_EMAIL},
|
||||
worker::WORKER_CONFIG,
|
||||
worker::{to_raw_value, WORKER_CONFIG},
|
||||
DB, METRICS_ENABLED,
|
||||
};
|
||||
|
||||
@@ -120,6 +131,7 @@ pub async fn cancel_job<'c: 'async_recursion>(
|
||||
&db,
|
||||
&job_running,
|
||||
format!("canceled by {username}: (force cancel: {force_cancel})"),
|
||||
job_running.mem_peak.unwrap_or(0),
|
||||
&e,
|
||||
None,
|
||||
rsmq.clone(),
|
||||
@@ -166,20 +178,39 @@ pub async fn cancel_job<'c: 'async_recursion>(
|
||||
Ok((tx, Some(id)))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct WrappedError<T: Serialize> {
|
||||
pub error: T,
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
pub async fn add_completed_job_error<
|
||||
T: Serialize + Send + Sync,
|
||||
R: rsmq_async::RsmqConnection + Clone + Send,
|
||||
>(
|
||||
db: &Pool<Postgres>,
|
||||
queued_job: &QueuedJob,
|
||||
logs: String,
|
||||
e: &serde_json::Value,
|
||||
mem_peak: i32,
|
||||
e: T,
|
||||
metrics: Option<Metrics>,
|
||||
rsmq: Option<R>,
|
||||
) -> Result<serde_json::Value, Error> {
|
||||
) -> Result<WrappedError<T>, Error> {
|
||||
if *METRICS_ENABLED {
|
||||
metrics.map(|m| m.worker_execution_failed.inc());
|
||||
}
|
||||
let result = serde_json::json!({ "error": e });
|
||||
let _ = add_completed_job(db, &queued_job, false, false, &result, logs, rsmq).await?;
|
||||
let result = WrappedError { error: e };
|
||||
let _ = add_completed_job(
|
||||
db,
|
||||
&queued_job,
|
||||
false,
|
||||
false,
|
||||
Json(&result),
|
||||
logs,
|
||||
mem_peak,
|
||||
rsmq,
|
||||
)
|
||||
.await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -206,13 +237,17 @@ lazy_static::lazy_static! {
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip_all, name = "add_completed_job")]
|
||||
pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
pub async fn add_completed_job<
|
||||
T: Serialize + Send + Sync,
|
||||
R: rsmq_async::RsmqConnection + Clone + Send,
|
||||
>(
|
||||
db: &Pool<Postgres>,
|
||||
queued_job: &QueuedJob,
|
||||
success: bool,
|
||||
skipped: bool,
|
||||
result: &serde_json::Value,
|
||||
result: Json<&T>,
|
||||
logs: String,
|
||||
mem_peak: i32,
|
||||
rsmq: Option<R>,
|
||||
) -> Result<Uuid, Error> {
|
||||
// tracing::error!("Start");
|
||||
@@ -245,14 +280,11 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
None
|
||||
};
|
||||
|
||||
let mem_peak = sqlx::query_scalar!("SELECT mem_peak FROM queue WHERE id = $1", &queued_job.id)
|
||||
.fetch_optional(db)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.flatten();
|
||||
let mut tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
|
||||
let job_id = queued_job.id.clone();
|
||||
let job_id = queued_job.id;
|
||||
// tracing::error!("1 {:?}", start.elapsed());
|
||||
|
||||
let mem_peak = mem_peak.max(queued_job.mem_peak.unwrap_or(0));
|
||||
let _duration: i64 = sqlx::query_scalar!(
|
||||
"INSERT INTO completed_job AS cj
|
||||
( workspace_id
|
||||
@@ -298,8 +330,8 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
success,
|
||||
queued_job.script_hash.map(|x| x.0),
|
||||
queued_job.script_path,
|
||||
queued_job.args,
|
||||
result,
|
||||
&queued_job.args as &Option<Json<HashMap<String, Box<RawValue>>>>,
|
||||
result as Json<&T>,
|
||||
logs,
|
||||
queued_job.raw_code,
|
||||
queued_job.raw_lock,
|
||||
@@ -317,15 +349,18 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
duration as Option<i64>,
|
||||
queued_job.email,
|
||||
queued_job.visible_to_owner,
|
||||
mem_peak,
|
||||
if mem_peak > 0 { Some(mem_peak) } else { None },
|
||||
queued_job.tag,
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e}")))?;
|
||||
// tracing::error!("2 {:?}", start.elapsed());
|
||||
|
||||
// tracing::error!("Added completed job {:#?}", queued_job);
|
||||
tx = delete_job(tx, &queued_job.workspace_id, job_id).await?;
|
||||
// tracing::error!("3 {:?}", start.elapsed());
|
||||
|
||||
if !queued_job.is_flow_step
|
||||
&& queued_job.schedule_path.is_some()
|
||||
&& queued_job.script_path.is_some()
|
||||
@@ -337,7 +372,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
queued_job.script_path.as_ref().unwrap(),
|
||||
&queued_job.workspace_id,
|
||||
success,
|
||||
&result,
|
||||
result,
|
||||
job_id,
|
||||
queued_job.started_at.unwrap_or(chrono::Utc::now()),
|
||||
)
|
||||
@@ -399,7 +434,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
&& queued_job.parent_job.is_none()
|
||||
&& !success
|
||||
{
|
||||
if let Err(e) = send_error_to_global_handler(rsmq.clone(), &queued_job, db, &result).await {
|
||||
if let Err(e) = send_error_to_global_handler(rsmq.clone(), &queued_job, db, result).await {
|
||||
tracing::error!(
|
||||
"Could not run global error handler for job {}: {}",
|
||||
&queued_job.id,
|
||||
@@ -407,8 +442,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
);
|
||||
}
|
||||
|
||||
if let Err(e) =
|
||||
send_error_to_workspace_handler(rsmq.clone(), &queued_job, db, &result).await
|
||||
if let Err(e) = send_error_to_workspace_handler(rsmq.clone(), &queued_job, db, result).await
|
||||
{
|
||||
tracing::error!(
|
||||
"Could not run workspace error handler for job {}: {}",
|
||||
@@ -419,16 +453,20 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
}
|
||||
|
||||
tracing::debug!("Added completed job {}", queued_job.id);
|
||||
// tracing::error!("{:?}", start.elapsed());
|
||||
// tracing::error!("4 {:?}", start.elapsed());
|
||||
|
||||
Ok(queued_job.id)
|
||||
}
|
||||
|
||||
pub async fn run_error_handler<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
pub async fn run_error_handler<
|
||||
'a,
|
||||
T: Serialize + Send + Sync,
|
||||
R: rsmq_async::RsmqConnection + Clone + Send,
|
||||
>(
|
||||
rsmq: Option<R>,
|
||||
queued_job: &QueuedJob,
|
||||
db: &Pool<Postgres>,
|
||||
result: &serde_json::Value,
|
||||
result: Json<&'a T>,
|
||||
error_handler_path: &str,
|
||||
error_handler_extra_args: Option<serde_json::Value>,
|
||||
is_global: bool,
|
||||
@@ -437,16 +475,22 @@ pub async fn run_error_handler<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
let script_w_id = if is_global { "admins" } else { w_id }; // script workspace id
|
||||
let job_id = queued_job.id;
|
||||
let (job_payload, tag) = script_path_to_payload(&error_handler_path, db, script_w_id).await?;
|
||||
let mut args = result.as_object().unwrap().clone();
|
||||
args.insert("workspace_id".to_string(), json!(w_id));
|
||||
args.insert("job_id".to_string(), json!(job_id));
|
||||
args.insert("path".to_string(), json!(queued_job.script_path));
|
||||
args.insert("is_flow".to_string(), json!(queued_job.raw_flow.is_some()));
|
||||
args.insert("email".to_string(), json!(queued_job.email));
|
||||
|
||||
let mut extra = HashMap::new();
|
||||
extra.insert("workspace_id".to_string(), to_raw_value(&w_id));
|
||||
extra.insert("job_id".to_string(), to_raw_value(&job_id));
|
||||
extra.insert("path".to_string(), to_raw_value(&queued_job.script_path));
|
||||
extra.insert(
|
||||
"is_flow".to_string(),
|
||||
to_raw_value(&queued_job.raw_flow.is_some()),
|
||||
);
|
||||
extra.insert("email".to_string(), to_raw_value(&queued_job.email));
|
||||
|
||||
if let Some(extra_args) = error_handler_extra_args {
|
||||
if let serde_json::Value::Object(args_m) = extra_args {
|
||||
args.extend(args_m);
|
||||
for (k, v) in args_m {
|
||||
extra.insert(k, to_raw_value(&v));
|
||||
}
|
||||
} else {
|
||||
return Err(error::Error::ExecutionErr(
|
||||
"args of scripts needs to be dict".to_string(),
|
||||
@@ -461,7 +505,7 @@ pub async fn run_error_handler<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
tx,
|
||||
script_w_id,
|
||||
job_payload,
|
||||
args,
|
||||
PushArgs { extra, args: result.to_owned() },
|
||||
if is_global { "global" } else { "error_handler" },
|
||||
if is_global {
|
||||
SUPERADMIN_SECRET_EMAIL
|
||||
@@ -497,11 +541,15 @@ pub async fn run_error_handler<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn send_error_to_global_handler<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
pub async fn send_error_to_global_handler<
|
||||
'a,
|
||||
T: Serialize + Send + Sync,
|
||||
R: rsmq_async::RsmqConnection + Clone + Send,
|
||||
>(
|
||||
rsmq: Option<R>,
|
||||
queued_job: &QueuedJob,
|
||||
db: &Pool<Postgres>,
|
||||
result: &serde_json::Value,
|
||||
result: Json<&'a T>,
|
||||
) -> Result<(), Error> {
|
||||
if let Some(ref global_error_handler) = *GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE {
|
||||
run_error_handler(
|
||||
@@ -519,11 +567,16 @@ pub async fn send_error_to_global_handler<R: rsmq_async::RsmqConnection + Clone
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn send_error_to_workspace_handler<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
pub async fn send_error_to_workspace_handler<
|
||||
'a,
|
||||
'c,
|
||||
T: Serialize + Send + Sync,
|
||||
R: rsmq_async::RsmqConnection + Clone + Send,
|
||||
>(
|
||||
rsmq: Option<R>,
|
||||
queued_job: &QueuedJob,
|
||||
db: &Pool<Postgres>,
|
||||
result: &serde_json::Value,
|
||||
result: Json<&'a T>,
|
||||
) -> Result<(), Error> {
|
||||
let w_id = &queued_job.workspace_id;
|
||||
let mut tx = db.begin().await?;
|
||||
@@ -620,20 +673,25 @@ pub async fn handle_maybe_scheduled_job<'c, R: rsmq_async::RsmqConnection + Clon
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
#[derive(Clone, Serialize)]
|
||||
struct CompletedJobSubset {
|
||||
success: bool,
|
||||
result: Option<serde_json::Value>,
|
||||
started_at: chrono::DateTime<chrono::Utc>,
|
||||
}
|
||||
async fn apply_schedule_handlers<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
|
||||
async fn apply_schedule_handlers<
|
||||
'a,
|
||||
'c,
|
||||
T: Serialize + Send + Sync,
|
||||
R: rsmq_async::RsmqConnection + Clone + Send + 'c,
|
||||
>(
|
||||
mut tx: QueueTransaction<'c, R>,
|
||||
db: &Pool<Postgres>,
|
||||
schedule_path: &str,
|
||||
script_path: &str,
|
||||
w_id: &str,
|
||||
success: bool,
|
||||
result: &serde_json::Value,
|
||||
result: Json<&'a T>,
|
||||
job_id: Uuid,
|
||||
started_at: DateTime<Utc>,
|
||||
) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
|
||||
@@ -790,7 +848,12 @@ async fn apply_schedule_handlers<'c, R: rsmq_async::RsmqConnection + Clone + Sen
|
||||
Ok(tx)
|
||||
}
|
||||
|
||||
pub async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
|
||||
pub async fn handle_on_failure<
|
||||
'a,
|
||||
'c,
|
||||
T: Serialize + Send + Sync,
|
||||
R: rsmq_async::RsmqConnection + Clone + Send + 'c,
|
||||
>(
|
||||
db: &Pool<Postgres>,
|
||||
tx: QueueTransaction<'c, R>,
|
||||
schedule_path: &str,
|
||||
@@ -798,7 +861,7 @@ pub async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send
|
||||
is_flow: bool,
|
||||
w_id: &str,
|
||||
on_failure_path: &str,
|
||||
result: &serde_json::Value,
|
||||
result: Json<&'a T>,
|
||||
failed_times: i32,
|
||||
started_at: DateTime<Utc>,
|
||||
extra_args: Option<serde_json::Value>,
|
||||
@@ -808,16 +871,18 @@ pub async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send
|
||||
) -> windmill_common::error::Result<(Uuid, QueueTransaction<'c, R>)> {
|
||||
let (payload, tag) = get_payload_tag_from_prefixed_path(on_failure_path, db, w_id).await?;
|
||||
|
||||
let mut args = result.clone().as_object().unwrap().clone();
|
||||
args.insert("schedule_path".to_string(), json!(schedule_path));
|
||||
args.insert("path".to_string(), json!(script_path));
|
||||
args.insert("is_flow".to_string(), json!(is_flow));
|
||||
args.insert("started_at".to_string(), json!(started_at));
|
||||
args.insert("failed_times".to_string(), json!(failed_times));
|
||||
let mut extra = HashMap::new();
|
||||
extra.insert("schedule_path".to_string(), to_raw_value(&schedule_path));
|
||||
extra.insert("path".to_string(), to_raw_value(&script_path));
|
||||
extra.insert("is_flow".to_string(), to_raw_value(&is_flow));
|
||||
extra.insert("started_at".to_string(), to_raw_value(&started_at));
|
||||
extra.insert("failed_times".to_string(), to_raw_value(&failed_times));
|
||||
|
||||
if let Some(args_v) = extra_args {
|
||||
if let serde_json::Value::Object(args_m) = args_v {
|
||||
args.extend(args_m);
|
||||
for (k, v) in args_m {
|
||||
extra.insert(k, to_raw_value(&v));
|
||||
}
|
||||
} else {
|
||||
return Err(error::Error::ExecutionErr(
|
||||
"args of scripts needs to be dict".to_string(),
|
||||
@@ -831,7 +896,7 @@ pub async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send
|
||||
tx,
|
||||
w_id,
|
||||
payload,
|
||||
args,
|
||||
PushArgs { extra, args: result.to_owned() },
|
||||
username,
|
||||
email,
|
||||
permissioned_as,
|
||||
@@ -857,7 +922,20 @@ pub async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send
|
||||
return Ok((uuid, tx));
|
||||
}
|
||||
|
||||
async fn handle_on_recovery<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
|
||||
// #[derive(Serialize)]
|
||||
// pub struct RecoveryValue<T> {
|
||||
// error_started_at: chrono::DateTime<Utc>,
|
||||
// schedule_path: String,
|
||||
// path: String,
|
||||
// is_flow: boolean,
|
||||
// extra_args: serde_json::Value
|
||||
// }
|
||||
async fn handle_on_recovery<
|
||||
'a,
|
||||
'c,
|
||||
T: Serialize + Send + Sync,
|
||||
R: rsmq_async::RsmqConnection + Clone + Send + 'c,
|
||||
>(
|
||||
db: &Pool<Postgres>,
|
||||
tx: QueueTransaction<'c, R>,
|
||||
schedule_path: &str,
|
||||
@@ -866,7 +944,7 @@ async fn handle_on_recovery<'c, R: rsmq_async::RsmqConnection + Clone + Send + '
|
||||
w_id: &str,
|
||||
on_recovery_path: &str,
|
||||
error_job: CompletedJobSubset,
|
||||
successful_job_result: &serde_json::Value,
|
||||
successful_job_result: Json<&'a T>,
|
||||
successful_times: i32,
|
||||
successful_job_started_at: DateTime<Utc>,
|
||||
extra_args: Option<serde_json::Value>,
|
||||
@@ -886,7 +964,11 @@ async fn handle_on_recovery<'c, R: rsmq_async::RsmqConnection + Clone + Send + '
|
||||
args.insert("schedule_path".to_string(), json!(schedule_path));
|
||||
args.insert("path".to_string(), json!(script_path));
|
||||
args.insert("is_flow".to_string(), json!(is_flow));
|
||||
args.insert("success_result".to_string(), successful_job_result.clone());
|
||||
args.insert(
|
||||
"success_result".to_string(),
|
||||
serde_json::from_str(&serde_json::to_string(&successful_job_result).unwrap())
|
||||
.unwrap_or_else(|_| json!("{}")),
|
||||
);
|
||||
args.insert("success_times".to_string(), json!(successful_times));
|
||||
args.insert(
|
||||
"success_started_at".to_string(),
|
||||
@@ -1074,14 +1156,14 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
|
||||
// if using redis, only one message at a time can be poped from the queue. Process only this message and move to the next elligible job
|
||||
// In this case, the job might be a job from the same script path, but we can't optimise this further
|
||||
// if using posgtres, then we're able to re-queue the entire batch of scheduled job for this script_path, so we do it
|
||||
let _requeued_job = sqlx::query_as::<_, QueuedJob>(&format!(
|
||||
let requeued_job_tag = sqlx::query_scalar::<_, String>(&format!(
|
||||
"UPDATE queue
|
||||
SET running = false
|
||||
, started_at = null
|
||||
, scheduled_for = '{estimated_next_schedule_timestamp}'
|
||||
, logs = CASE WHEN logs IS NULL OR logs = '' THEN '{job_log_event}'::text WHEN logs LIKE '%{job_log_event}' THEN logs ELSE concat(logs, '{job_log_line_break}{job_log_event}'::text) END
|
||||
WHERE id = '{job_uuid}'
|
||||
RETURNING *"
|
||||
RETURNING tag"
|
||||
))
|
||||
.fetch_one(&mut tx)
|
||||
.await
|
||||
@@ -1091,20 +1173,19 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
|
||||
rsmq.send_message(
|
||||
job_uuid.to_bytes_le().to_vec(),
|
||||
Option::Some(estimated_next_schedule_timestamp),
|
||||
_requeued_job.tag,
|
||||
requeued_job_tag,
|
||||
);
|
||||
}
|
||||
tx.commit().await?;
|
||||
} else {
|
||||
// if using posgtres, then we're able to re-queue the entire batch of scheduled job for this script_path, so we do it
|
||||
let _requeued_jobs = sqlx::query_as::<_, QueuedJob>(&format!(
|
||||
sqlx::query(&format!(
|
||||
"UPDATE queue
|
||||
SET running = false
|
||||
, started_at = null
|
||||
, scheduled_for = '{estimated_next_schedule_timestamp}'
|
||||
, logs = CASE WHEN logs IS NULL OR logs = '' THEN '{job_log_event}'::text WHEN logs LIKE '%{job_log_event}' THEN logs ELSE concat(logs, '{job_log_line_break}{job_log_event}'::text) END
|
||||
WHERE (id = '{job_uuid}') OR (script_path = '{job_script_path}' AND running = false AND scheduled_for <= now())
|
||||
RETURNING *"
|
||||
WHERE (id = '{job_uuid}') OR (script_path = '{job_script_path}' AND running = false AND scheduled_for <= now())"
|
||||
))
|
||||
.fetch_all(&mut tx)
|
||||
.await
|
||||
@@ -1154,7 +1235,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
.map_err(|_| anyhow::anyhow!("Failed to parsed Redis message"))?,
|
||||
);
|
||||
|
||||
let m2 = sqlx::query_as::<_, QueuedJob>(
|
||||
let m2r = sqlx::query(
|
||||
"UPDATE queue
|
||||
SET running = true
|
||||
, started_at = coalesce(started_at, now())
|
||||
@@ -1166,6 +1247,11 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
.bind(uuid)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
let m2 = if let Some(row) = m2r {
|
||||
Some(QueuedJob::from_row(&row)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
rsmq.delete_message(&tag.unwrap(), &msg.id)
|
||||
.await
|
||||
@@ -1190,7 +1276,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
let tags = config.worker_tags.clone();
|
||||
drop(config);
|
||||
let r = if suspend_first {
|
||||
sqlx::query_as::<_, QueuedJob>("UPDATE queue
|
||||
sqlx::query("UPDATE queue
|
||||
SET running = true
|
||||
, started_at = coalesce(started_at, now())
|
||||
, last_ping = now()
|
||||
@@ -1210,14 +1296,18 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let r = if let Some(row) = r {
|
||||
Some(QueuedJob::from_row(&row)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if r.is_none() {
|
||||
// #[cfg(feature = "benchmark")]
|
||||
// let instant = Instant::now();
|
||||
|
||||
let tags = WORKER_CONFIG.read().await.worker_tags.clone();
|
||||
|
||||
let r = sqlx::query_as::<_, QueuedJob>(
|
||||
let r = sqlx::query(
|
||||
"UPDATE queue
|
||||
SET running = true
|
||||
, started_at = coalesce(started_at, now())
|
||||
@@ -1239,7 +1329,11 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
// #[cfg(feature = "benchmark")]
|
||||
// println!("pull query: {:?}", instant.elapsed());
|
||||
|
||||
r
|
||||
if let Some(row) = r {
|
||||
Some(QueuedJob::from_row(&row)?)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
r
|
||||
}
|
||||
@@ -1247,13 +1341,19 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
Ok(job)
|
||||
}
|
||||
|
||||
#[derive(FromRow)]
|
||||
struct ResultR {
|
||||
result: Option<Json<Box<RawValue>>>,
|
||||
}
|
||||
|
||||
#[async_recursion]
|
||||
pub async fn get_result_by_id(
|
||||
db: Pool<Postgres>,
|
||||
w_id: String,
|
||||
flow_id: Uuid,
|
||||
node_id: String,
|
||||
) -> error::Result<serde_json::Value> {
|
||||
json_path: Option<String>,
|
||||
) -> error::Result<Box<RawValue>> {
|
||||
let flow_job_result = sqlx::query!(
|
||||
"SELECT leaf_jobs->$1::text as leaf_jobs, parent_job FROM queue WHERE COALESCE((SELECT root_job FROM queue WHERE id = $2), $2) = id AND workspace_id = $3",
|
||||
node_id,
|
||||
@@ -1265,7 +1365,7 @@ pub async fn get_result_by_id(
|
||||
|
||||
let flow_job_result = windmill_common::utils::not_found_if_none(
|
||||
flow_job_result,
|
||||
"Flow result by id",
|
||||
"Flow result by id in leaf jobs",
|
||||
format!("{}, {}", flow_id, node_id),
|
||||
)?;
|
||||
|
||||
@@ -1281,7 +1381,7 @@ pub async fn get_result_by_id(
|
||||
.await?
|
||||
.flatten()
|
||||
.unwrap_or(parent_job);
|
||||
return get_result_by_id(db, w_id, root_job, node_id).await;
|
||||
return get_result_by_id(db, w_id, root_job, node_id, json_path).await;
|
||||
}
|
||||
|
||||
let result_id = windmill_common::utils::not_found_if_none(
|
||||
@@ -1292,27 +1392,37 @@ pub async fn get_result_by_id(
|
||||
|
||||
let value = match result_id {
|
||||
JobResult::ListJob(x) => {
|
||||
let rows = sqlx::query_scalar!(
|
||||
let rows = sqlx::query(
|
||||
"SELECT result FROM completed_job WHERE id = ANY($1) AND workspace_id = $2",
|
||||
x.as_slice(),
|
||||
w_id,
|
||||
)
|
||||
.bind(x.as_slice())
|
||||
.bind(w_id)
|
||||
.fetch_all(&db)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter_map(|x| x)
|
||||
.collect::<Vec<serde_json::Value>>();
|
||||
serde_json::json!(rows)
|
||||
.filter_map(|x| ResultR::from_row(&x).ok().and_then(|x| x.result))
|
||||
.collect::<Vec<Json<Box<RawValue>>>>();
|
||||
to_raw_value(&rows)
|
||||
}
|
||||
JobResult::SingleJob(x) => sqlx::query_scalar!(
|
||||
"SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
x,
|
||||
w_id,
|
||||
JobResult::SingleJob(x) => sqlx::query(
|
||||
"SELECT result #> $3 as result FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(x)
|
||||
.bind(w_id)
|
||||
.bind(
|
||||
json_path
|
||||
.map(|x| x.split(".").map(|x| x.to_string()).collect::<Vec<_>>())
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?
|
||||
.map(|r| {
|
||||
ResultR::from_row(&r)
|
||||
.ok()
|
||||
.and_then(|x| x.result.map(|x| x.0))
|
||||
})
|
||||
.flatten()
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)),
|
||||
};
|
||||
|
||||
Ok(value)
|
||||
@@ -1366,7 +1476,7 @@ pub async fn get_queued_job<'c>(
|
||||
w_id: &str,
|
||||
tx: &mut Transaction<'c, Postgres>,
|
||||
) -> error::Result<Option<QueuedJob>> {
|
||||
let r = sqlx::query_as::<_, QueuedJob>(
|
||||
let r = sqlx::query(
|
||||
"SELECT *
|
||||
FROM queue WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
@@ -1374,7 +1484,11 @@ pub async fn get_queued_job<'c>(
|
||||
.bind(w_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await?;
|
||||
Ok(r)
|
||||
if let Some(row) = r {
|
||||
Ok(Some(QueuedJob::from_row(&row)?.to_owned()))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
pub enum PushIsolationLevel<'c, R: rsmq_async::RsmqConnection + Send + 'c> {
|
||||
@@ -1407,13 +1521,144 @@ macro_rules! fetch_scalar_isolated {
|
||||
};
|
||||
}
|
||||
|
||||
use sqlx::types::JsonRawValue;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct PushArgs<T> {
|
||||
#[serde(flatten)]
|
||||
pub extra: HashMap<String, Box<RawValue>>,
|
||||
#[serde(flatten)]
|
||||
pub args: Json<T>,
|
||||
}
|
||||
|
||||
#[axum::async_trait]
|
||||
impl<S> FromRequest<S, axum::body::Body> for PushArgs<HashMap<String, Box<RawValue>>>
|
||||
where
|
||||
S: Send + Sync,
|
||||
{
|
||||
type Rejection = Response;
|
||||
|
||||
async fn from_request(
|
||||
req: Request<axum::body::Body>,
|
||||
_state: &S,
|
||||
) -> Result<Self, Self::Rejection> {
|
||||
let (content_type, mut extra, use_raw) = {
|
||||
let headers_map = req.headers();
|
||||
let content_type_header = headers_map.get(CONTENT_TYPE);
|
||||
let content_type = content_type_header.and_then(|value| value.to_str().ok());
|
||||
(
|
||||
content_type,
|
||||
build_extra(&headers_map),
|
||||
req.uri().query().is_some_and(|x| x.contains("raw=true")),
|
||||
)
|
||||
};
|
||||
|
||||
if content_type.is_none() || content_type.unwrap().starts_with("application/json") {
|
||||
let bytes = Bytes::from_request(req, _state)
|
||||
.await
|
||||
.map_err(IntoResponse::into_response)?;
|
||||
let str = String::from_utf8(bytes.to_vec())
|
||||
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
|
||||
|
||||
if use_raw {
|
||||
extra.insert("raw_string".to_string(), to_raw_value(&str));
|
||||
}
|
||||
|
||||
let wrap_body = str.len() > 0 && str.chars().next().unwrap() != '{';
|
||||
|
||||
if wrap_body {
|
||||
let args = serde_json::from_str::<Option<Box<RawValue>>>(&str)
|
||||
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())?
|
||||
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
|
||||
let mut hm = HashMap::new();
|
||||
hm.insert("body".to_string(), args);
|
||||
Ok(PushArgs { extra, args: Json(hm) })
|
||||
} else {
|
||||
let hm = serde_json::from_str::<Option<HashMap<String, Box<JsonRawValue>>>>(&str)
|
||||
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())?
|
||||
.unwrap_or_else(HashMap::new);
|
||||
Ok(PushArgs { extra, args: Json(hm) })
|
||||
}
|
||||
} else if content_type
|
||||
.unwrap()
|
||||
.starts_with("application/x-www-form-urlencoded")
|
||||
{
|
||||
let Form(payload): Form<Option<HashMap<String, Box<RawValue>>>> =
|
||||
req.extract().await.map_err(IntoResponse::into_response)?;
|
||||
return Ok(PushArgs {
|
||||
extra: HashMap::new(),
|
||||
args: Json(payload.unwrap_or_else(HashMap::new)),
|
||||
});
|
||||
} else {
|
||||
Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref INCLUDE_HEADERS: Vec<String> = std::env::var("INCLUDE_HEADERS")
|
||||
.ok().map(|x| x
|
||||
.split(',')
|
||||
.map(|s| s.to_string())
|
||||
.collect()).unwrap_or_default();
|
||||
}
|
||||
|
||||
pub fn build_extra(headers: &HeaderMap) -> HashMap<String, Box<RawValue>> {
|
||||
let mut args = HashMap::new();
|
||||
let whitelist = headers
|
||||
.get("include_header")
|
||||
.map(|s| {
|
||||
s.to_str()
|
||||
.unwrap_or_default()
|
||||
.split(",")
|
||||
.map(|s| s.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
whitelist
|
||||
.iter()
|
||||
.chain(INCLUDE_HEADERS.iter())
|
||||
.for_each(|h| {
|
||||
if let Some(v) = headers.get(h) {
|
||||
args.insert(
|
||||
h.to_string().to_lowercase().replace('-', "_"),
|
||||
to_raw_value(&v.to_str().unwrap().to_string()),
|
||||
);
|
||||
}
|
||||
});
|
||||
args
|
||||
}
|
||||
|
||||
impl PushArgs<HashMap<String, Box<RawValue>>> {
|
||||
pub fn empty() -> Self {
|
||||
PushArgs { extra: HashMap::new(), args: Json(HashMap::new()) }
|
||||
}
|
||||
}
|
||||
|
||||
pub fn empty_args() -> Box<RawValue> {
|
||||
return JsonRawValue::from_string("{}".to_string()).unwrap();
|
||||
}
|
||||
|
||||
impl From<HashMap<String, Box<JsonRawValue>>> for PushArgs<HashMap<String, Box<JsonRawValue>>> {
|
||||
fn from(value: HashMap<String, Box<JsonRawValue>>) -> Self {
|
||||
PushArgs { extra: HashMap::new(), args: Json(value) }
|
||||
}
|
||||
}
|
||||
|
||||
// impl<T> From<PushArgsInner<T>> for PushArgs<T> {
|
||||
// fn from(value: PushArgsInner<T>) -> Self {
|
||||
// PushArgs::Unwrapped(value)
|
||||
// }
|
||||
// }
|
||||
|
||||
// #[instrument(level = "trace", skip_all)]
|
||||
pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
_db: &Pool<Postgres>,
|
||||
mut tx: PushIsolationLevel<'c, R>,
|
||||
workspace_id: &str,
|
||||
job_payload: JobPayload,
|
||||
args: serde_json::Map<String, serde_json::Value>,
|
||||
args: T,
|
||||
user: &str,
|
||||
email: &str,
|
||||
permissioned_as: String,
|
||||
@@ -1430,8 +1675,6 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
custom_timeout: Option<i32>,
|
||||
flow_step_id: Option<String>,
|
||||
) -> Result<(Uuid, QueueTransaction<'c, R>), Error> {
|
||||
let args_json = serde_json::Value::Object(args);
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
{
|
||||
let premium_workspace = *CLOUD_HOSTED
|
||||
@@ -1835,7 +2078,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
script_path.clone(),
|
||||
raw_code,
|
||||
raw_lock,
|
||||
args_json,
|
||||
Json(args) as Json<T>,
|
||||
job_kind.clone() as JobKind,
|
||||
schedule_path,
|
||||
raw_flow.map(|f| serde_json::json!(f)),
|
||||
@@ -1852,7 +2095,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
concurrency_time_window_s,
|
||||
custom_timeout,
|
||||
flow_step_id,
|
||||
cache_ttl
|
||||
cache_ttl,
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
.await
|
||||
|
||||
@@ -23,7 +23,6 @@ windmill-common = { workspace = true, features = [
|
||||
"prometheus",
|
||||
"tracing_init",
|
||||
] }
|
||||
windmill-api-client.workspace = true
|
||||
windmill-parser.workspace = true
|
||||
windmill-parser-ts.workspace = true
|
||||
windmill-parser-go.workspace = true
|
||||
@@ -73,6 +72,7 @@ pem = { workspace = true, optional = true }
|
||||
urlencoding.workspace = true
|
||||
nix.workspace = true
|
||||
bytes.workspace = true
|
||||
reqwest.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
deno_fetch.workspace = true
|
||||
|
||||
@@ -2,17 +2,18 @@ use std::{collections::HashMap, process::Stdio};
|
||||
|
||||
use itertools::Itertools;
|
||||
use regex::Regex;
|
||||
use serde_json::{json, Value};
|
||||
use serde_json::{json, value::RawValue};
|
||||
use sqlx::types::Json;
|
||||
use tokio::process::Command;
|
||||
use windmill_common::{error::Error, jobs::QueuedJob};
|
||||
use windmill_common::{error::Error, jobs::QueuedJob, worker::to_raw_value};
|
||||
|
||||
const BIN_BASH: &str = "/bin/bash";
|
||||
const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto");
|
||||
|
||||
use crate::{
|
||||
common::{
|
||||
get_reserved_variables, handle_child, read_file, read_file_content, set_logs,
|
||||
start_child_process, transform_json_value, write_file,
|
||||
build_args_map, get_reserved_variables, handle_child, read_file, read_file_content,
|
||||
set_logs, start_child_process, write_file,
|
||||
},
|
||||
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
|
||||
TZ_ENV,
|
||||
@@ -26,6 +27,7 @@ lazy_static::lazy_static! {
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn handle_bash_job(
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job: &QueuedJob,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
@@ -35,7 +37,7 @@ pub async fn handle_bash_job(
|
||||
base_internal_url: &str,
|
||||
worker_name: &str,
|
||||
envs: HashMap<String, String>,
|
||||
) -> Result<serde_json::Value, Error> {
|
||||
) -> Result<Box<RawValue>, Error> {
|
||||
logs.push_str("\n\n--- BASH CODE EXECUTION ---\n");
|
||||
set_logs(logs, &job.id, db).await;
|
||||
write_file(
|
||||
@@ -48,29 +50,19 @@ pub async fn handle_bash_job(
|
||||
let mut reserved_variables = get_reserved_variables(job, &token, db).await?;
|
||||
reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
|
||||
|
||||
let client = client.get_authed().await;
|
||||
let hm = match transform_json_value(
|
||||
"args",
|
||||
&client,
|
||||
&job.workspace_id,
|
||||
job.args.clone().unwrap_or_else(|| json!({})),
|
||||
job,
|
||||
db,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Value::Object(ref hm) => hm.clone(),
|
||||
_ => serde_json::Map::new(),
|
||||
let args = build_args_map(job, client, db).await?.map(Json);
|
||||
let job_args = if args.is_some() {
|
||||
args.as_ref()
|
||||
} else {
|
||||
job.args.as_ref()
|
||||
};
|
||||
|
||||
let args_owned = windmill_parser_bash::parse_bash_sig(&content)?
|
||||
.args
|
||||
.iter()
|
||||
.map(|arg| {
|
||||
hm.get(&arg.name)
|
||||
.and_then(|v| match v {
|
||||
Value::String(s) => Some(s.clone()),
|
||||
_ => serde_json::to_string(v).ok(),
|
||||
})
|
||||
job_args
|
||||
.and_then(|x| x.get(&arg.name).map(|x| raw_to_string(x.get())))
|
||||
.unwrap_or_else(String::new)
|
||||
})
|
||||
.collect::<Vec<String>>();
|
||||
@@ -122,6 +114,7 @@ pub async fn handle_bash_job(
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child,
|
||||
!*DISABLE_NSJAIL,
|
||||
worker_name,
|
||||
@@ -143,7 +136,7 @@ pub async fn handle_bash_job(
|
||||
if let Ok(metadata) = tokio::fs::metadata(&result_out_path).await {
|
||||
if metadata.len() > 0 {
|
||||
let result = read_file_content(&result_out_path).await?;
|
||||
return Ok(json!(result));
|
||||
return Ok(to_raw_value(&json!(result)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,12 +146,20 @@ pub async fn handle_bash_job(
|
||||
.last()
|
||||
.map(|x| ANSI_ESCAPE_RE.replace_all(x, "").to_string())
|
||||
.unwrap_or_else(String::new));
|
||||
Ok(last_line)
|
||||
Ok(to_raw_value(&last_line))
|
||||
}
|
||||
|
||||
fn raw_to_string(x: &str) -> String {
|
||||
match serde_json::from_str::<serde_json::Value>(x) {
|
||||
Ok(serde_json::Value::String(x)) => x,
|
||||
Ok(x) => serde_json::to_string(&x).unwrap_or_else(|_| String::new()),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn handle_powershell_job(
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job: &QueuedJob,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
@@ -168,36 +169,25 @@ pub async fn handle_powershell_job(
|
||||
base_internal_url: &str,
|
||||
worker_name: &str,
|
||||
envs: HashMap<String, String>,
|
||||
) -> Result<serde_json::Value, Error> {
|
||||
) -> Result<Box<RawValue>, Error> {
|
||||
logs.push_str("\n\n--- POWERSHELL CODE EXECUTION ---\n");
|
||||
set_logs(logs, &job.id, db).await;
|
||||
let pwsh_args = {
|
||||
let client = client.get_authed().await;
|
||||
let hm = match transform_json_value(
|
||||
"args",
|
||||
&client,
|
||||
&job.workspace_id,
|
||||
job.args.clone().unwrap_or_else(|| json!({})),
|
||||
job,
|
||||
db,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Value::Object(ref hm) => hm.clone(),
|
||||
_ => serde_json::Map::new(),
|
||||
let args = build_args_map(job, client, db).await?.map(Json);
|
||||
let job_args = if args.is_some() {
|
||||
args.as_ref()
|
||||
} else {
|
||||
job.args.as_ref()
|
||||
};
|
||||
|
||||
let args_owned = windmill_parser_bash::parse_powershell_sig(&content)?
|
||||
let args_owned = windmill_parser_bash::parse_bash_sig(&content)?
|
||||
.args
|
||||
.iter()
|
||||
.map(|arg| {
|
||||
(
|
||||
arg.name.clone(),
|
||||
hm.get(&arg.name)
|
||||
.and_then(|v| match v {
|
||||
Value::String(s) => Some(s.clone()),
|
||||
_ => serde_json::to_string(v).ok(),
|
||||
})
|
||||
job_args
|
||||
.and_then(|x| x.get(&arg.name).map(|x| raw_to_string(x.get())))
|
||||
.unwrap_or_else(String::new),
|
||||
)
|
||||
})
|
||||
@@ -262,6 +252,7 @@ pub async fn handle_powershell_job(
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child,
|
||||
!*DISABLE_NSJAIL,
|
||||
worker_name,
|
||||
@@ -277,5 +268,5 @@ pub async fn handle_powershell_job(
|
||||
.last()
|
||||
.map(|x| ANSI_ESCAPE_RE.replace_all(x, "").to_string())
|
||||
.unwrap_or_else(String::new));
|
||||
Ok(last_line)
|
||||
Ok(to_raw_value(&last_line))
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
use serde_json::{json, Value};
|
||||
use windmill_common::error::Error;
|
||||
use serde_json::{json, value::RawValue, Value};
|
||||
use windmill_common::jobs::QueuedJob;
|
||||
use windmill_common::{error::Error, worker::to_raw_value};
|
||||
use windmill_parser_sql::parse_bigquery_sig;
|
||||
use windmill_queue::HTTP_CLIENT;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{common::transform_json_value, AuthedClient};
|
||||
use crate::{common::build_args_values, AuthedClientBackgroundTask};
|
||||
|
||||
use gcp_auth::{AuthenticationManager, CustomServiceAccount};
|
||||
|
||||
@@ -52,19 +52,12 @@ struct BigqueryError {
|
||||
}
|
||||
|
||||
pub async fn do_bigquery(
|
||||
job: QueuedJob,
|
||||
client: &AuthedClient,
|
||||
job: &QueuedJob,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
query: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
) -> windmill_common::error::Result<serde_json::Value> {
|
||||
let args = if let Some(args) = &job.args {
|
||||
Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let bigquery_args: Value = serde_json::from_value(args.unwrap_or_else(|| json!({})))
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
) -> windmill_common::error::Result<Box<RawValue>> {
|
||||
let bigquery_args = build_args_values(job, client, db).await?;
|
||||
|
||||
let database = bigquery_args
|
||||
.get("database")
|
||||
@@ -85,14 +78,6 @@ pub async fn do_bigquery(
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
|
||||
let args = &job
|
||||
.args
|
||||
.clone()
|
||||
.unwrap_or_else(|| json!({}))
|
||||
.as_object()
|
||||
.map(|x| x.to_owned())
|
||||
.unwrap_or_else(|| json!({}).as_object().unwrap().to_owned());
|
||||
|
||||
let mut statement_values: Vec<Value> = vec![];
|
||||
|
||||
let sig = parse_bigquery_sig(&query)
|
||||
@@ -102,7 +87,7 @@ pub async fn do_bigquery(
|
||||
for arg in &sig {
|
||||
let arg_t = arg.otyp.clone().unwrap_or_else(|| "string".to_string());
|
||||
let arg_n = arg.clone().name;
|
||||
let arg_v = args.get(&arg.name).cloned().unwrap_or(json!(""));
|
||||
let arg_v = bigquery_args.get(&arg.name).cloned().unwrap_or(json!(""));
|
||||
let bigquery_v = if arg_t.ends_with("[]") {
|
||||
let base_type = arg_t.strip_suffix("[]").unwrap_or(&arg_t);
|
||||
json!({
|
||||
@@ -114,7 +99,7 @@ pub async fn do_bigquery(
|
||||
}
|
||||
},
|
||||
"parameterValue": {
|
||||
"arrayValues": args
|
||||
"arrayValues": bigquery_args
|
||||
.get(&arg.name)
|
||||
.unwrap_or(&json!([]))
|
||||
.as_array()
|
||||
@@ -177,7 +162,7 @@ pub async fn do_bigquery(
|
||||
}
|
||||
|
||||
if result.rows.is_none() || result.rows.as_ref().unwrap().len() == 0 {
|
||||
return Ok(Value::Array(vec![]));
|
||||
return Ok(serde_json::from_str("[]").unwrap());
|
||||
}
|
||||
|
||||
if result.schema.is_none() {
|
||||
@@ -217,9 +202,9 @@ pub async fn do_bigquery(
|
||||
});
|
||||
Value::from(row_map)
|
||||
})
|
||||
.collect();
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
return Ok(rows);
|
||||
return Ok(to_raw_value(&rows));
|
||||
}
|
||||
Err(e) => match response.json::<BigqueryErrorResponse>().await {
|
||||
Ok(bq_err) => return Err(Error::ExecutionErr(bq_err.error.message)),
|
||||
|
||||
@@ -9,6 +9,7 @@ use anyhow::Context;
|
||||
use base64::Engine;
|
||||
use itertools::Itertools;
|
||||
use regex::Regex;
|
||||
use serde_json::value::RawValue;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
@@ -63,6 +64,7 @@ lazy_static::lazy_static! {
|
||||
|
||||
pub async fn gen_lockfile(
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job_id: &Uuid,
|
||||
w_id: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
@@ -109,6 +111,7 @@ pub async fn gen_lockfile(
|
||||
job_id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child_process,
|
||||
false,
|
||||
worker_name,
|
||||
@@ -152,6 +155,7 @@ pub async fn gen_lockfile(
|
||||
|
||||
install_lockfile(
|
||||
logs,
|
||||
mem_peak,
|
||||
job_id,
|
||||
w_id,
|
||||
db,
|
||||
@@ -187,6 +191,7 @@ pub async fn gen_lockfile(
|
||||
|
||||
pub async fn install_lockfile(
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job_id: &Uuid,
|
||||
w_id: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
@@ -208,6 +213,7 @@ pub async fn install_lockfile(
|
||||
job_id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child_process,
|
||||
false,
|
||||
worker_name,
|
||||
@@ -241,6 +247,7 @@ pub fn get_trusted_deps(code: &str) -> Vec<String> {
|
||||
pub async fn handle_bun_job(
|
||||
requirements_o: Option<String>,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job: &QueuedJob,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
@@ -250,7 +257,7 @@ pub async fn handle_bun_job(
|
||||
worker_name: &str,
|
||||
envs: HashMap<String, String>,
|
||||
shared_mount: &str,
|
||||
) -> error::Result<serde_json::Value> {
|
||||
) -> error::Result<Box<RawValue>> {
|
||||
let _ = write_file(job_dir, "main.ts", inner_content).await?;
|
||||
|
||||
let common_bun_proc_envs: HashMap<String, String> =
|
||||
@@ -283,6 +290,7 @@ pub async fn handle_bun_job(
|
||||
|
||||
install_lockfile(
|
||||
logs,
|
||||
mem_peak,
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
db,
|
||||
@@ -305,6 +313,7 @@ pub async fn handle_bun_job(
|
||||
set_logs(&logs, &job.id, &db).await;
|
||||
let _ = gen_lockfile(
|
||||
logs,
|
||||
mem_peak,
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
db,
|
||||
@@ -385,12 +394,12 @@ run().catch(async (e) => {{
|
||||
};
|
||||
|
||||
let reserved_variables_args_out_f = async {
|
||||
let client = client.get_authed().await;
|
||||
let args_and_out_f = async {
|
||||
create_args_and_out_file(&client, job, job_dir, db).await?;
|
||||
Ok(()) as Result<()>
|
||||
};
|
||||
let reserved_variables_f = async {
|
||||
let client = client.get_authed().await;
|
||||
let vars = get_reserved_variables(job, &client.token, db).await?;
|
||||
Ok(vars) as Result<HashMap<String, String>>
|
||||
};
|
||||
@@ -491,6 +500,7 @@ plugin(p)
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child,
|
||||
false,
|
||||
worker_name,
|
||||
@@ -528,6 +538,9 @@ pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String
|
||||
return bun_envs;
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn start_worker(
|
||||
requirements_o: Option<String>,
|
||||
@@ -541,7 +554,7 @@ pub async fn start_worker(
|
||||
script_path: &str,
|
||||
token: &str,
|
||||
job_completed_tx: Sender<JobCompleted>,
|
||||
mut jobs_rx: Receiver<QueuedJob>,
|
||||
mut jobs_rx: Receiver<Arc<QueuedJob>>,
|
||||
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
|
||||
) -> Result<()> {
|
||||
use std::task::Poll;
|
||||
@@ -549,6 +562,7 @@ pub async fn start_worker(
|
||||
use futures::{future, Future};
|
||||
|
||||
let mut logs = "".to_string();
|
||||
let mut mem_peak: i32 = 0;
|
||||
let _ = write_file(job_dir, "main.ts", inner_content).await?;
|
||||
let common_bun_proc_envs: HashMap<String, String> =
|
||||
get_common_bun_proc_envs(&base_internal_url).await;
|
||||
@@ -591,6 +605,7 @@ pub async fn start_worker(
|
||||
}
|
||||
install_lockfile(
|
||||
&mut logs,
|
||||
&mut mem_peak,
|
||||
&Uuid::nil(),
|
||||
&w_id,
|
||||
db,
|
||||
@@ -604,6 +619,7 @@ pub async fn start_worker(
|
||||
logs.push_str("\n\n--- BUN INSTALL ---\n");
|
||||
let _ = gen_lockfile(
|
||||
&mut logs,
|
||||
&mut mem_peak,
|
||||
&Uuid::nil(),
|
||||
&w_id,
|
||||
db,
|
||||
@@ -807,8 +823,8 @@ plugin(p)
|
||||
tracing::debug!("processed job");
|
||||
|
||||
let result = serde_json::from_str(&line).expect("json is ok");
|
||||
let job: QueuedJob = jobs.pop_front().expect("pop");
|
||||
job_completed_tx.send(JobCompleted { job , result, logs: "".to_string(), success: true, cached_res_path: None, token: token.to_string() }).await.unwrap();
|
||||
let job: Arc<QueuedJob> = jobs.pop_front().expect("pop");
|
||||
job_completed_tx.send(JobCompleted { job , result, logs: "".to_string(), mem_peak: 0, success: true, cached_res_path: None, token: token.to_string() }).await.unwrap();
|
||||
} else {
|
||||
tracing::info!("dedicated worker process exited");
|
||||
break;
|
||||
@@ -820,7 +836,7 @@ plugin(p)
|
||||
tracing::debug!("received job");
|
||||
jobs.push_back(job.clone());
|
||||
// write_stdin(&mut stdin, &serde_json::to_string(&job.args.unwrap_or_else(|| serde_json::json!({"x": job.id}))).expect("serialize")).await?;
|
||||
write_stdin(&mut stdin, &serde_json::to_string(&job.args.unwrap_or_else(|| serde_json::json!({}))).expect("serialize")).await?;
|
||||
write_stdin(&mut stdin, &serde_json::to_string(&job.args).expect("serialize")).await?;
|
||||
stdin.flush().await.context("stdin flush")?;
|
||||
} else {
|
||||
tracing::debug!("job channel closed");
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
use async_recursion::async_recursion;
|
||||
use itertools::Itertools;
|
||||
use nix::sys::signal::{self, Signal};
|
||||
use nix::unistd::Pid;
|
||||
use regex::Regex;
|
||||
use serde::Serialize;
|
||||
use serde_json::value::RawValue;
|
||||
use serde_json::{json, Value};
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::process::Command;
|
||||
use tokio::{fs::File, io::AsyncReadExt};
|
||||
use windmill_api_client::{types::CreateResource, Client};
|
||||
use windmill_common::worker::CLOUD_HOSTED;
|
||||
use windmill_common::{
|
||||
error::{self, Error},
|
||||
@@ -41,24 +44,60 @@ use futures::{
|
||||
};
|
||||
|
||||
use crate::{
|
||||
AuthedClient, MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGTERM, ROOT_CACHE_DIR, TIMEOUT_DURATION,
|
||||
WHITELIST_ENVS,
|
||||
AuthedClient, AuthedClientBackgroundTask, MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGTERM,
|
||||
ROOT_CACHE_DIR, TIMEOUT_DURATION, WHITELIST_ENVS,
|
||||
};
|
||||
|
||||
pub async fn build_args_map<'a>(
|
||||
job: &'a QueuedJob,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
db: &Pool<Postgres>,
|
||||
) -> error::Result<Option<HashMap<String, Box<RawValue>>>> {
|
||||
if let Some(args) = &job.args {
|
||||
return transform_json(client, &job.workspace_id, &args.0, &job, db).await;
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
pub async fn build_args_values(
|
||||
job: &QueuedJob,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
db: &Pool<Postgres>,
|
||||
) -> error::Result<HashMap<String, serde_json::Value>> {
|
||||
if let Some(args) = &job.args {
|
||||
transform_json_as_values(client, &job.workspace_id, &args.0, &job, db).await
|
||||
} else {
|
||||
Ok(HashMap::new())
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn create_args_and_out_file(
|
||||
client: &AuthedClient,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
job: &QueuedJob,
|
||||
job_dir: &str,
|
||||
db: &Pool<Postgres>,
|
||||
) -> Result<(), Error> {
|
||||
let args = if let Some(args) = &job.args {
|
||||
Some(transform_json_value("args", client, &job.workspace_id, args.clone(), job, db).await?)
|
||||
if let Some(args) = &job.args {
|
||||
if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, db).await? {
|
||||
write_file(
|
||||
job_dir,
|
||||
"args.json",
|
||||
&serde_json::to_string(&x).unwrap_or_else(|_| "{}".to_string()),
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
write_file(
|
||||
job_dir,
|
||||
"args.json",
|
||||
&serde_json::to_string(&args).unwrap_or_else(|_| "{}".to_string()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
} else {
|
||||
None
|
||||
write_file(job_dir, "args.json", "{}").await?;
|
||||
};
|
||||
let ser_args = serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
write_file(job_dir, "args.json", &ser_args).await?;
|
||||
|
||||
write_file(job_dir, "result.json", "").await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -81,6 +120,80 @@ pub async fn write_file_binary(dir: &str, path: &str, content: &[u8]) -> error::
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref RE_RES_VAR: Regex = Regex::new(r#"\$(?:var|res)\:"#).unwrap();
|
||||
}
|
||||
|
||||
pub async fn transform_json<'a>(
|
||||
client: &AuthedClientBackgroundTask,
|
||||
workspace: &str,
|
||||
vs: &'a HashMap<String, Box<RawValue>>,
|
||||
job: &QueuedJob,
|
||||
db: &Pool<Postgres>,
|
||||
) -> error::Result<Option<HashMap<String, Box<RawValue>>>> {
|
||||
let mut has_match = false;
|
||||
for (_, v) in vs {
|
||||
let inner_vs = v.get();
|
||||
if (*RE_RES_VAR).is_match(inner_vs) {
|
||||
has_match = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !has_match {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut r = HashMap::new();
|
||||
for (k, v) in vs {
|
||||
let inner_vs = v.get();
|
||||
if (*RE_RES_VAR).is_match(inner_vs) {
|
||||
let value = serde_json::from_str(inner_vs).map_err(|e| {
|
||||
error::Error::InternalErr(format!("Error while parsing inner arg: {e}"))
|
||||
})?;
|
||||
let transformed =
|
||||
transform_json_value(&k, &client.get_authed().await, workspace, value, job, db)
|
||||
.await?;
|
||||
let as_raw = serde_json::from_value(transformed).map_err(|e| {
|
||||
error::Error::InternalErr(format!("Error while parsing inner arg: {e}"))
|
||||
})?;
|
||||
r.insert(k.to_string(), as_raw);
|
||||
} else {
|
||||
r.insert(k.to_string(), v.to_owned());
|
||||
}
|
||||
}
|
||||
Ok(Some(r))
|
||||
}
|
||||
|
||||
pub async fn transform_json_as_values<'a>(
|
||||
client: &AuthedClientBackgroundTask,
|
||||
workspace: &str,
|
||||
vs: &'a HashMap<String, Box<RawValue>>,
|
||||
job: &QueuedJob,
|
||||
db: &Pool<Postgres>,
|
||||
) -> error::Result<HashMap<String, serde_json::Value>> {
|
||||
let mut r: HashMap<String, serde_json::Value> = HashMap::new();
|
||||
for (k, v) in vs {
|
||||
let inner_vs = v.get();
|
||||
if (*RE_RES_VAR).is_match(inner_vs) {
|
||||
let value = serde_json::from_str(inner_vs).map_err(|e| {
|
||||
error::Error::InternalErr(format!("Error while parsing inner arg: {e}"))
|
||||
})?;
|
||||
let transformed =
|
||||
transform_json_value(&k, &client.get_authed().await, workspace, value, job, db)
|
||||
.await?;
|
||||
let as_raw = serde_json::from_value(transformed).map_err(|e| {
|
||||
error::Error::InternalErr(format!("Error while parsing inner arg: {e}"))
|
||||
})?;
|
||||
r.insert(k.to_string(), as_raw);
|
||||
} else {
|
||||
r.insert(
|
||||
k.to_string(),
|
||||
serde_json::from_str(v.get()).unwrap_or_else(|_| serde_json::Value::Null),
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
#[async_recursion]
|
||||
pub async fn transform_json_value(
|
||||
name: &str,
|
||||
@@ -94,11 +207,10 @@ pub async fn transform_json_value(
|
||||
Value::String(y) if y.starts_with("$var:") => {
|
||||
let path = y.strip_prefix("$var:").unwrap();
|
||||
client
|
||||
.get_client()
|
||||
.get_variable_value(workspace, path)
|
||||
.get_variable_value(path)
|
||||
.await
|
||||
.map(|x| json!(x))
|
||||
.map_err(|_| Error::NotFound(format!("Variable {path} not found for `{name}`")))
|
||||
.map(|v| json!(v.into_inner()))
|
||||
}
|
||||
Value::String(y) if y.starts_with("$res:") => {
|
||||
let path = y.strip_prefix("$res:").unwrap();
|
||||
@@ -107,12 +219,13 @@ pub async fn transform_json_value(
|
||||
"Argument `{name}` is an invalid resource path: {path}",
|
||||
)));
|
||||
}
|
||||
Ok(client
|
||||
.get_client()
|
||||
.get_resource_value_interpolated(workspace, path, Some(&job.id))
|
||||
client
|
||||
.get_resource_value_interpolated::<serde_json::Value>(
|
||||
path,
|
||||
Some(job.id.to_string()),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::NotFound(format!("Resource {path} not found for `{name}`")))?
|
||||
.into_inner())
|
||||
.map_err(|_| Error::NotFound(format!("Resource {path} not found for `{name}`")))
|
||||
}
|
||||
Value::String(y) if y.starts_with("$") => {
|
||||
let flow_path = if let Some(uuid) = job.parent_job {
|
||||
@@ -167,30 +280,31 @@ pub async fn read_file_content(path: &str) -> error::Result<String> {
|
||||
file.read_to_string(&mut content).await?;
|
||||
Ok(content)
|
||||
}
|
||||
pub async fn read_file(path: &str) -> error::Result<serde_json::Value> {
|
||||
// tracing::error!("START1");
|
||||
// let start = Instant::now();
|
||||
|
||||
let r = if *CLOUD_HOSTED {
|
||||
let content = read_file_content(path).await?;
|
||||
if content.len() > MAX_RESULT_SIZE {
|
||||
return Err(error::Error::ExecutionErr("Result is too large for the cloud app (limit 2MB).
|
||||
If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned()));
|
||||
};
|
||||
serde_json::from_str(&content)
|
||||
.map_err(|e| error::Error::ExecutionErr(format!("Error parsing result: {e}")))
|
||||
} else {
|
||||
let file = std::fs::File::open(path)
|
||||
.map_err(|e| error::Error::ExecutionErr(format!("Error opening file {path}: {e}")))?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
|
||||
serde_json::from_reader(reader)
|
||||
.map_err(|e| error::Error::ExecutionErr(format!("Error parsing result: {e}")))
|
||||
};
|
||||
// tracing::error!("{:?}", start.elapsed());
|
||||
return r;
|
||||
pub async fn read_file_bytes(path: &str) -> error::Result<Vec<u8>> {
|
||||
let mut file = File::open(path).await?;
|
||||
let mut content = Vec::new();
|
||||
file.read_to_end(&mut content).await?;
|
||||
Ok(content)
|
||||
}
|
||||
pub async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
|
||||
|
||||
//this skips more steps than from_str at the cost of being unsafe. The source must ALWAUS gemerate valid json or this can cause UB in the worst case
|
||||
pub fn unsafe_raw(json: String) -> Box<RawValue> {
|
||||
unsafe { std::mem::transmute::<Box<str>, Box<RawValue>>(json.into()) }
|
||||
}
|
||||
|
||||
pub async fn read_file(path: &str) -> error::Result<Box<RawValue>> {
|
||||
let content = read_file_content(path).await?;
|
||||
|
||||
if *CLOUD_HOSTED && content.len() > MAX_RESULT_SIZE {
|
||||
return Err(error::Error::ExecutionErr("Result is too large for the cloud app (limit 2MB).
|
||||
If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned()));
|
||||
};
|
||||
|
||||
let r = unsafe_raw(content);
|
||||
return Ok(r);
|
||||
}
|
||||
pub async fn read_result(job_dir: &str) -> error::Result<Box<RawValue>> {
|
||||
return read_file(&format!("{job_dir}/result.json")).await;
|
||||
}
|
||||
|
||||
@@ -302,6 +416,7 @@ pub async fn handle_child(
|
||||
job_id: &Uuid,
|
||||
db: &Pool<Postgres>,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
mut child: Child,
|
||||
nsjail: bool,
|
||||
worker_name: &str,
|
||||
@@ -343,6 +458,7 @@ pub async fn handle_child(
|
||||
interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
|
||||
|
||||
let mut i = 0;
|
||||
|
||||
loop {
|
||||
tokio::select!(
|
||||
_ = rx.recv() => break,
|
||||
@@ -358,10 +474,12 @@ pub async fn handle_child(
|
||||
.await
|
||||
.expect("update worker ping");
|
||||
}
|
||||
let mem_peak = get_mem_peak(pid, nsjail).await;
|
||||
tracing::info!("{job_id} in {} still running. mem peak: {}kB", _w_id, mem_peak);
|
||||
let mem_peak = if mem_peak > 0 { Some(mem_peak) } else { None };
|
||||
if sqlx::query_scalar!("UPDATE queue SET mem_peak = GREATEST($1, mem_peak), last_ping = now() WHERE id = $2 RETURNING canceled", mem_peak, job_id)
|
||||
let current_mem = get_mem_peak(pid, nsjail).await;
|
||||
if current_mem > *mem_peak {
|
||||
*mem_peak = current_mem
|
||||
}
|
||||
tracing::info!("{job_id} in {_w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB");
|
||||
if sqlx::query_scalar!("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled", *mem_peak, job_id)
|
||||
.fetch_optional(&db)
|
||||
.await
|
||||
.map(|v| Some(true) == v)
|
||||
@@ -570,7 +688,7 @@ pub async fn handle_child(
|
||||
|
||||
let (wait_result, _) = tokio::join!(wait_on_child, lines);
|
||||
|
||||
tracing::info!(%job_id, "child process '{child_name}' for {job_id} took {}ms", start.elapsed().as_millis());
|
||||
tracing::info!(%job_id, "child process '{child_name}' for {job_id} took {}ms, mem_peak: {:?}", start.elapsed().as_millis(), mem_peak);
|
||||
match wait_result {
|
||||
_ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
|
||||
"logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)"
|
||||
@@ -657,27 +775,49 @@ fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn hash_args(v: &serde_json::Value) -> i64 {
|
||||
let mut dh = DefaultHasher::new();
|
||||
serde_json::to_string(v).unwrap().hash(&mut dh);
|
||||
dh.finish() as i64
|
||||
pub fn hash_args(v: &Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>) -> i64 {
|
||||
if let Some(vs) = v {
|
||||
let mut dh = DefaultHasher::new();
|
||||
let hm = &vs.0;
|
||||
for k in hm.keys().sorted() {
|
||||
k.hash(&mut dh);
|
||||
hm.get(k).unwrap().get().hash(&mut dh);
|
||||
}
|
||||
dh.finish() as i64
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn save_in_cache(client: &AuthedClient, job: &QueuedJob, cached_path: String, r: &Value) {
|
||||
let client: &Client = client.get_client();
|
||||
#[derive(Serialize)]
|
||||
struct StoreCachedResource<'a> {
|
||||
expire: i64,
|
||||
value: &'a RawValue,
|
||||
}
|
||||
|
||||
pub async fn save_in_cache<'a>(
|
||||
db: &Pool<Postgres>,
|
||||
job: &QueuedJob,
|
||||
cached_path: String,
|
||||
r: &'a RawValue,
|
||||
) {
|
||||
let expire = chrono::Utc::now().timestamp() + job.cache_ttl.unwrap() as i64;
|
||||
let cr = &CreateResource {
|
||||
path: cached_path,
|
||||
description: None,
|
||||
resource_type: "cache".to_string(),
|
||||
value: serde_json::json!({
|
||||
"value": r,
|
||||
"expire": expire
|
||||
}),
|
||||
};
|
||||
if let Err(e) = client
|
||||
.create_resource(&job.workspace_id, Some(true), cr)
|
||||
.await
|
||||
|
||||
let store_cache_resource = StoreCachedResource { expire, value: r };
|
||||
let raw_json = sqlx::types::Json(store_cache_resource);
|
||||
|
||||
if let Err(e) = sqlx::query!(
|
||||
"INSERT INTO resource
|
||||
(workspace_id, path, value, resource_type)
|
||||
VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path)
|
||||
DO UPDATE SET value = $3",
|
||||
job.workspace_id,
|
||||
cached_path,
|
||||
raw_json as sqlx::types::Json<StoreCachedResource>,
|
||||
"cache"
|
||||
)
|
||||
.execute(db)
|
||||
.await
|
||||
{
|
||||
tracing::error!("Error creating cache resource {e}")
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::{collections::HashMap, process::Stdio};
|
||||
|
||||
use itertools::Itertools;
|
||||
use serde_json::value::RawValue;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
@@ -73,6 +74,7 @@ pub async fn generate_deno_lock(
|
||||
job_id: &Uuid,
|
||||
code: &str,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job_dir: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
w_id: &str,
|
||||
@@ -117,6 +119,7 @@ pub async fn generate_deno_lock(
|
||||
job_id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child_process,
|
||||
false,
|
||||
worker_name,
|
||||
@@ -138,6 +141,7 @@ pub async fn generate_deno_lock(
|
||||
pub async fn handle_deno_job(
|
||||
requirements_o: Option<String>,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job: &QueuedJob,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
@@ -146,7 +150,7 @@ pub async fn handle_deno_job(
|
||||
base_internal_url: &str,
|
||||
worker_name: &str,
|
||||
envs: HashMap<String, String>,
|
||||
) -> error::Result<serde_json::Value> {
|
||||
) -> error::Result<Box<RawValue>> {
|
||||
// let mut start = Instant::now();
|
||||
logs.push_str("\n\n--- DENO CODE EXECUTION ---\n");
|
||||
|
||||
@@ -247,18 +251,18 @@ run().catch(async (e) => {{
|
||||
};
|
||||
|
||||
let reserved_variables_args_out_f = async {
|
||||
let client = client.get_authed().await;
|
||||
let args_and_out_f = async {
|
||||
create_args_and_out_file(&client, job, job_dir, db).await?;
|
||||
Ok(()) as Result<()>
|
||||
};
|
||||
let reserved_variables_f = async {
|
||||
let client = client.get_authed().await;
|
||||
let mut vars = get_reserved_variables(job, &client.token, db).await?;
|
||||
vars.insert("RUST_LOG".to_string(), "info".to_string());
|
||||
Ok(vars) as Result<HashMap<String, String>>
|
||||
Ok((vars, client.token)) as Result<(HashMap<String, String>, String)>
|
||||
};
|
||||
let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?;
|
||||
Ok((reserved_variables, client.token)) as error::Result<(HashMap<String, String>, String)>
|
||||
Ok(reserved_variables) as error::Result<(HashMap<String, String>, String)>
|
||||
};
|
||||
|
||||
let (_, (reserved_variables, token), _, _, _) = tokio::try_join!(
|
||||
@@ -325,6 +329,7 @@ run().catch(async (e) => {{
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child,
|
||||
false,
|
||||
worker_name,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::{collections::HashMap, process::Stdio};
|
||||
|
||||
use itertools::Itertools;
|
||||
use serde_json::value::RawValue;
|
||||
use tokio::{
|
||||
fs::{DirBuilder, File},
|
||||
io::AsyncReadExt,
|
||||
@@ -33,6 +34,7 @@ lazy_static::lazy_static! {
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn handle_go_job(
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job: &QueuedJob,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
@@ -43,7 +45,7 @@ pub async fn handle_go_job(
|
||||
base_internal_url: &str,
|
||||
worker_name: &str,
|
||||
envs: HashMap<String, String>,
|
||||
) -> Result<serde_json::Value, Error> {
|
||||
) -> Result<Box<RawValue>, Error> {
|
||||
//go does not like executing modules at temp root
|
||||
let job_dir = &format!("{job_dir}/go");
|
||||
let bin_path = if let Some(requirements) = requirements_o.clone() {
|
||||
@@ -68,8 +70,6 @@ pub async fn handle_go_job(
|
||||
(false, false)
|
||||
};
|
||||
|
||||
let client = &client.get_authed().await;
|
||||
|
||||
if !bin_exists {
|
||||
logs.push_str("\n\n--- GO DEPENDENCIES SETUP ---\n");
|
||||
set_logs(logs, &job.id, db).await;
|
||||
@@ -78,6 +78,7 @@ pub async fn handle_go_job(
|
||||
&job.id,
|
||||
inner_content,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
true,
|
||||
@@ -193,6 +194,7 @@ func Run(req Req) (interface{{}}, error){{
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
build_go_process,
|
||||
false,
|
||||
worker_name,
|
||||
@@ -216,6 +218,8 @@ func Run(req Req) (interface{{}}, error){{
|
||||
create_args_and_out_file(client, job, job_dir, db).await?;
|
||||
}
|
||||
|
||||
let client = &client.get_authed().await;
|
||||
|
||||
let reserved_variables = get_reserved_variables(job, &client.token, db).await?;
|
||||
|
||||
let child = if !*DISABLE_NSJAIL {
|
||||
@@ -270,6 +274,7 @@ func Run(req Req) (interface{{}}, error){{
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child,
|
||||
!*DISABLE_NSJAIL,
|
||||
worker_name,
|
||||
@@ -307,6 +312,7 @@ pub async fn install_go_dependencies(
|
||||
job_id: &Uuid,
|
||||
code: &str,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job_dir: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
non_dep_job: bool,
|
||||
@@ -329,6 +335,7 @@ pub async fn install_go_dependencies(
|
||||
job_id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child_process,
|
||||
false,
|
||||
worker_name,
|
||||
@@ -392,6 +399,7 @@ pub async fn install_go_dependencies(
|
||||
job_id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child_process,
|
||||
false,
|
||||
worker_name,
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::{json, Value};
|
||||
use serde_json::{json, value::RawValue};
|
||||
use sqlx::types::Json;
|
||||
use windmill_common::error::Error;
|
||||
use windmill_common::jobs::QueuedJob;
|
||||
use windmill_queue::HTTP_CLIENT;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{common::transform_json_value, AuthedClient};
|
||||
use crate::{common::build_args_map, AuthedClientBackgroundTask};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GraphqlApi {
|
||||
@@ -18,7 +19,7 @@ struct GraphqlApi {
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GraphqlResponse {
|
||||
data: Option<Value>,
|
||||
data: Option<Box<RawValue>>,
|
||||
errors: Option<Vec<GraphqlError>>,
|
||||
}
|
||||
|
||||
@@ -28,34 +29,28 @@ struct GraphqlError {
|
||||
}
|
||||
|
||||
pub async fn do_graphql(
|
||||
job: QueuedJob,
|
||||
client: &AuthedClient,
|
||||
job: &QueuedJob,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
query: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
) -> windmill_common::error::Result<serde_json::Value> {
|
||||
let args = if let Some(args) = &job.args {
|
||||
Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
|
||||
) -> windmill_common::error::Result<Box<RawValue>> {
|
||||
let args = build_args_map(job, client, db).await?.map(Json);
|
||||
let job_args = if args.is_some() {
|
||||
args.as_ref()
|
||||
} else {
|
||||
None
|
||||
job.args.as_ref()
|
||||
};
|
||||
|
||||
let graphql_args: serde_json::Value = serde_json::from_value(args.unwrap_or_else(|| json!({})))
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
let api =
|
||||
serde_json::from_value::<GraphqlApi>(graphql_args.get("api").unwrap_or(&json!({})).clone())
|
||||
.map_err(|e: serde_json::Error| Error::ExecutionErr(e.to_string()))?;
|
||||
|
||||
let args = &job
|
||||
.args
|
||||
.clone()
|
||||
.unwrap_or_else(|| json!({}))
|
||||
.as_object()
|
||||
.map(|x| x.to_owned())
|
||||
.unwrap_or_else(|| json!({}).as_object().unwrap().to_owned());
|
||||
let api = if let Some(db) = job_args.as_ref().and_then(|x| x.get("api")) {
|
||||
serde_json::from_str::<GraphqlApi>(db.get())
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?
|
||||
} else {
|
||||
return Err(Error::BadRequest("Missing api argument".to_string()));
|
||||
};
|
||||
|
||||
let mut request = HTTP_CLIENT.post(api.base_url).json(&json!({
|
||||
"query": query,
|
||||
"variables": args
|
||||
"variables": job_args
|
||||
}));
|
||||
|
||||
if let Some(token) = &api.bearer_token {
|
||||
@@ -89,5 +84,7 @@ pub async fn do_graphql(
|
||||
}
|
||||
|
||||
// And then check that we got back the same string we sent over.
|
||||
return Ok(result.data.unwrap_or(json!({})));
|
||||
return Ok(result
|
||||
.data
|
||||
.unwrap_or_else(|| serde_json::from_str("{}").unwrap()));
|
||||
}
|
||||
|
||||
@@ -20,7 +20,8 @@ use deno_web::{BlobStore, TimersPermission};
|
||||
use itertools::Itertools;
|
||||
use lazy_static::lazy_static;
|
||||
use regex::Regex;
|
||||
use serde_json::Value;
|
||||
use serde_json::value::RawValue;
|
||||
use sqlx::types::Json;
|
||||
use tokio::{
|
||||
sync::{mpsc, oneshot},
|
||||
time::timeout,
|
||||
@@ -28,7 +29,7 @@ use tokio::{
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{error::Error, flow_status::JobResult};
|
||||
|
||||
use crate::AuthedClient;
|
||||
use crate::{common::unsafe_raw, AuthedClient};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IdContext {
|
||||
@@ -68,12 +69,56 @@ impl TimersPermission for PermissionsContainer {
|
||||
}
|
||||
|
||||
pub struct OptAuthedClient(Option<AuthedClient>);
|
||||
|
||||
pub async fn eval_timeout(
|
||||
expr: String,
|
||||
env: Vec<(String, serde_json::Value)>,
|
||||
transform_context: HashMap<String, Arc<Box<RawValue>>>,
|
||||
flow_input: Option<Arc<HashMap<String, Box<RawValue>>>>,
|
||||
authed_client: Option<&AuthedClient>,
|
||||
by_id: Option<IdContext>,
|
||||
) -> anyhow::Result<serde_json::Value> {
|
||||
) -> anyhow::Result<Box<RawValue>> {
|
||||
|
||||
let expr = expr.trim().to_string();
|
||||
|
||||
for (k,v) in transform_context.iter() {
|
||||
if k == &expr {
|
||||
return Ok(v.as_ref().clone())
|
||||
}
|
||||
}
|
||||
|
||||
if expr.starts_with("flow_input.") {
|
||||
if let Some(ref flow_input) = flow_input {
|
||||
for (k,v) in flow_input.iter() {
|
||||
if &format!("flow_input.{k}") == &expr {
|
||||
// tracing::error!("FLOW_INPUT");
|
||||
return Ok(v.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let p_id = by_id.as_ref().map(|x| format!("results.{}", x.previous_id));
|
||||
|
||||
if p_id.is_some() && transform_context.contains_key("previous_result") && &expr == p_id.as_ref().unwrap() {
|
||||
// tracing::error!("PREVIOUS_RESULT");
|
||||
return Ok(transform_context.get("previous_result").unwrap().as_ref().clone())
|
||||
}
|
||||
|
||||
if by_id.is_some() && authed_client.is_some() {
|
||||
if let Some(x) = RE_FULL.captures(&expr).and_then(|x| x.get(1).map(|y| y.as_str())) {
|
||||
// tracing::error!("{:?}", x.split(".").collect::<Vec<_>>());
|
||||
let arr = x.split(".").collect::<Vec<_>>();
|
||||
let mut iter = arr.iter();
|
||||
iter.next();
|
||||
if let Some(id) = iter.next() {
|
||||
let path = iter.join(".");
|
||||
let query = if path.is_empty() { None } else { Some(path) };
|
||||
return authed_client.unwrap().get_result_by_id(&by_id.as_ref().unwrap().flow_job.to_string(), id, query).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
let expr2 = expr.clone();
|
||||
let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
|
||||
let has_client = authed_client.is_some();
|
||||
@@ -81,7 +126,7 @@ pub async fn eval_timeout(
|
||||
timeout(
|
||||
std::time::Duration::from_millis(10000),
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let mut ops = vec![];
|
||||
let mut ops = vec![op_get_context::DECL];
|
||||
|
||||
if authed_client.is_some() {
|
||||
ops.extend([
|
||||
@@ -107,11 +152,35 @@ pub async fn eval_timeout(
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
let mut context_keys = transform_context
|
||||
.keys()
|
||||
.filter(|x|
|
||||
expr.contains(&x.to_string())
|
||||
)
|
||||
.map(|x| x.clone())
|
||||
.collect_vec();
|
||||
|
||||
if !context_keys.contains(&"previous_result".to_string()) && (p_id.is_some() && expr.contains(p_id.as_ref().unwrap())) || expr.contains("error") {
|
||||
context_keys.push("previous_result".to_string());
|
||||
}
|
||||
let has_flow_input = expr.contains("flow_input");
|
||||
if has_flow_input {
|
||||
context_keys.push("flow_input".to_string())
|
||||
}
|
||||
|
||||
let mut js_runtime = JsRuntime::new(options);
|
||||
{
|
||||
let op_state = js_runtime.op_state();
|
||||
let mut op_state = op_state.borrow_mut();
|
||||
op_state.put(OptAuthedClient(authed_client.clone()));
|
||||
op_state.put(TransformContext {
|
||||
flow_input: if has_flow_input { flow_input } else { None },
|
||||
envs: transform_context
|
||||
.into_iter()
|
||||
.filter(|(a, _)| context_keys.contains(a))
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
sender
|
||||
@@ -123,15 +192,21 @@ pub async fn eval_timeout(
|
||||
.build()?;
|
||||
|
||||
// pretty frail but this it to make the expr more user friendly and not require the user to write await
|
||||
let expr = ["variable", "step", "resource", "result_by_id"]
|
||||
let expr = ["variable", "resource"]
|
||||
.into_iter()
|
||||
.fold(expr, replace_with_await);
|
||||
|
||||
let expr = replace_with_await_result(expr);
|
||||
|
||||
let r = runtime.block_on(eval(&mut js_runtime, &expr, env, by_id, has_client))?;
|
||||
let r = runtime.block_on(eval(
|
||||
&mut js_runtime,
|
||||
&expr,
|
||||
context_keys,
|
||||
by_id,
|
||||
has_client,
|
||||
))?;
|
||||
|
||||
Ok(r) as anyhow::Result<Value>
|
||||
Ok(r) as anyhow::Result<Box<RawValue>>
|
||||
}),
|
||||
)
|
||||
.await
|
||||
@@ -155,7 +230,9 @@ fn replace_with_await(expr: String, fn_name: &str) -> String {
|
||||
s
|
||||
}
|
||||
lazy_static! {
|
||||
static ref RE: Regex = Regex::new("(?m)(?P<r>results.([a-z]|[A-Z]|_|[1-9])+)").unwrap();
|
||||
static ref RE: Regex = Regex::new(r"(?m)(?P<r>results\.(?:[a-z]|[A-Z]|_|[1-9])+)").unwrap();
|
||||
static ref RE_FULL: Regex = Regex::new(r"(?m)^results((?:\.(?:(?:[a-z]|[A-Z]|_|[1-9])+))+)$").unwrap();
|
||||
|
||||
}
|
||||
|
||||
fn replace_with_await_result(expr: String) -> String {
|
||||
@@ -184,10 +261,10 @@ fn add_closing_bracket(s: &str) -> String {
|
||||
async fn eval(
|
||||
context: &mut JsRuntime,
|
||||
expr: &str,
|
||||
env: Vec<(String, serde_json::Value)>,
|
||||
transform_context: Vec<String>,
|
||||
by_id: Option<IdContext>,
|
||||
has_client: bool,
|
||||
) -> anyhow::Result<serde_json::Value> {
|
||||
) -> anyhow::Result<Box<RawValue>> {
|
||||
let (api_code, by_id_code) = if has_client {
|
||||
let by_id_code = if let Some(by_id) = by_id {
|
||||
format!(
|
||||
@@ -205,12 +282,12 @@ async function result_by_id(node_id) {{
|
||||
}}
|
||||
}} else {{
|
||||
let flow_job_id = "{}";
|
||||
return await Deno.core.opAsync("op_get_id", [flow_job_id, node_id]);
|
||||
return JSON.parse(await Deno.core.opAsync("op_get_id", [flow_job_id, node_id]));
|
||||
}}
|
||||
}}
|
||||
|
||||
async function get_result(id) {{
|
||||
return await Deno.core.opAsync("op_get_result", [id]);
|
||||
return JSON.parse(await Deno.core.opAsync("op_get_result", [id]));
|
||||
}}
|
||||
const results = new Proxy({{}}, {{
|
||||
get: function(target, name, receiver) {{
|
||||
@@ -261,24 +338,30 @@ async function resource(path) {{
|
||||
};
|
||||
let code = format!(
|
||||
r#"
|
||||
function get_from_env(name) {{
|
||||
return JSON.parse(Deno.core.ops.op_get_context([name]));
|
||||
}}
|
||||
{api_code}
|
||||
{}
|
||||
{}
|
||||
{by_id_code}
|
||||
{HAS_CYCLE}
|
||||
((async () => {{
|
||||
{f};
|
||||
}})()).then((r) => hasCycle(r) ? 'cycle detected' : r)
|
||||
}})()).then((r) => hasCycle(r) ? 'cycle detected' : r).then(JSON.stringify)
|
||||
"#,
|
||||
env.into_iter()
|
||||
.map(|(a, b)| {
|
||||
format!(
|
||||
"let {a} = {};\n",
|
||||
serde_json::to_string(&b)
|
||||
.unwrap_or_else(|_| "\"error serializing value\"".to_string())
|
||||
)
|
||||
})
|
||||
transform_context
|
||||
.iter()
|
||||
.map(|a| { format!("let {a} = get_from_env(\"{a}\");\n",) })
|
||||
.join(""),
|
||||
if expr.contains("error") && transform_context.contains(&"previous_result".to_string()) {
|
||||
"let error = previous_result.error"
|
||||
} else {
|
||||
""
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
let global = context.execute_script("<anon>", code.into())?;
|
||||
let global = context.resolve_value(global).await?;
|
||||
|
||||
@@ -286,7 +369,8 @@ async function resource(path) {{
|
||||
let local = v8::Local::new(scope, global);
|
||||
// Deserialize a `v8` object into a Rust type using `serde_v8`,
|
||||
// in this case deserialize to a JSON `Value`.
|
||||
Ok(serde_v8::from_v8::<serde_json::Value>(scope, local)?)
|
||||
let r = serde_v8::from_v8::<String>(scope, local)?;
|
||||
Ok(unsafe_raw(r))
|
||||
}
|
||||
|
||||
const HAS_CYCLE: &str = r#"
|
||||
@@ -341,11 +425,7 @@ async fn op_variable(
|
||||
let path = &args[0];
|
||||
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
|
||||
if let Some(client) = client {
|
||||
let result = client
|
||||
.get_client()
|
||||
.get_variable_value(&client.workspace, path)
|
||||
.await?;
|
||||
Ok(result.into_inner())
|
||||
Ok(client.get_variable_value(path).await?)
|
||||
} else {
|
||||
anyhow::bail!("No client found in op state");
|
||||
}
|
||||
@@ -355,16 +435,15 @@ async fn op_variable(
|
||||
async fn op_get_result(
|
||||
op_state: Rc<RefCell<OpState>>,
|
||||
args: Vec<String>,
|
||||
) -> Result<serde_json::Value, anyhow::Error> {
|
||||
) -> Result<String, anyhow::Error> {
|
||||
let id = &args[0];
|
||||
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
|
||||
if let Some(client) = client {
|
||||
let result = client
|
||||
.get_client()
|
||||
.get_completed_job_result(&client.workspace, &id.parse()?)
|
||||
.get_completed_job_result::<Box<RawValue>>(id, None)
|
||||
.await?
|
||||
.clone();
|
||||
Ok(serde_json::json!(result))
|
||||
Ok(result.get().to_string())
|
||||
} else {
|
||||
anyhow::bail!("No client found in op state");
|
||||
}
|
||||
@@ -374,18 +453,16 @@ async fn op_get_result(
|
||||
async fn op_get_id(
|
||||
op_state: Rc<RefCell<OpState>>,
|
||||
args: Vec<String>,
|
||||
) -> Result<Option<serde_json::Value>, anyhow::Error> {
|
||||
) -> Result<Option<String>, anyhow::Error> {
|
||||
let flow_job_id = &args[0];
|
||||
let node_id = &args[1];
|
||||
|
||||
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
|
||||
if let Some(client) = client {
|
||||
let result = client
|
||||
.get_client()
|
||||
.result_by_id(&client.workspace, flow_job_id, node_id)
|
||||
.await
|
||||
.map_or(None, |e| Some(e.into_inner()));
|
||||
Ok(result)
|
||||
.get_result_by_id::<Option<Box<RawValue>>>(flow_job_id, node_id, None)
|
||||
.await?;
|
||||
Ok(result.map(|x| x.get().to_string()))
|
||||
} else {
|
||||
anyhow::bail!("No client found in op state");
|
||||
}
|
||||
@@ -400,16 +477,36 @@ async fn op_resource(
|
||||
|
||||
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
|
||||
if let Some(client) = client {
|
||||
let result = client
|
||||
.get_client()
|
||||
.get_resource_value_interpolated(&client.workspace, path, None)
|
||||
.await?;
|
||||
Ok(result.into_inner())
|
||||
client.get_resource_value_interpolated(path, None).await
|
||||
} else {
|
||||
anyhow::bail!("No client found in op state");
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TransformContext {
|
||||
pub envs: HashMap<String, Arc<Box<RawValue>>>,
|
||||
pub flow_input: Option<Arc<HashMap<String, Box<RawValue>>>>,
|
||||
}
|
||||
|
||||
#[op]
|
||||
fn op_get_context(op_state: Rc<RefCell<OpState>>, args: Vec<String>) -> String {
|
||||
let id = &args[0];
|
||||
let ops = op_state.borrow();
|
||||
let client = ops.borrow::<TransformContext>();
|
||||
if id == "flow_input" {
|
||||
return client
|
||||
.flow_input
|
||||
.as_ref()
|
||||
.and_then(|x| serde_json::to_string(&x).ok())
|
||||
.unwrap_or_else(|| "null".to_string());
|
||||
}
|
||||
return client
|
||||
.envs
|
||||
.get(id)
|
||||
.and_then(|x| serde_json::to_string(x).ok())
|
||||
.unwrap_or_else(String::new);
|
||||
}
|
||||
|
||||
pub fn transpile_ts(expr: String) -> anyhow::Result<String> {
|
||||
let parsed = deno_ast::parse_module(ParseParams {
|
||||
specifier: "eval.ts".to_string(),
|
||||
@@ -425,7 +522,7 @@ pub fn transpile_ts(expr: String) -> anyhow::Result<String> {
|
||||
static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin"));
|
||||
|
||||
pub struct MainArgs {
|
||||
args: Vec<serde_json::Value>,
|
||||
args: Vec<Option<Box<RawValue>>>,
|
||||
}
|
||||
|
||||
pub struct LogString {
|
||||
@@ -435,10 +532,14 @@ pub struct LogString {
|
||||
pub async fn eval_fetch_timeout(
|
||||
ts_expr: String,
|
||||
js_expr: String,
|
||||
args: serde_json::Map<String, Value>,
|
||||
) -> anyhow::Result<(serde_json::Value, String)> {
|
||||
args: Option<&Json<HashMap<String, Box<RawValue>>>>,
|
||||
) -> anyhow::Result<(Box<RawValue>, String)> {
|
||||
let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
|
||||
let ts_expr2 = ts_expr.clone();
|
||||
|
||||
let parsed_args = windmill_parser_ts::parse_deno_signature(&ts_expr, true)?.args;
|
||||
let spread = parsed_args.into_iter().map(|x| args.as_ref().and_then(|args| args.0.get(&x.name).map(|x| x.clone()))).collect::<Vec<_>>();
|
||||
|
||||
timeout(
|
||||
std::time::Duration::from_secs(100),
|
||||
tokio::task::spawn_blocking(move || {
|
||||
@@ -497,9 +598,7 @@ pub async fn eval_fetch_timeout(
|
||||
return y*2;
|
||||
});
|
||||
|
||||
let parsed_args = windmill_parser_ts::parse_deno_signature(&ts_expr, true)?.args;
|
||||
let spread = parsed_args.into_iter().map(|x| args.get(&x.name).map(|x| x.clone()).unwrap_or(serde_json::Value::Null)).collect::<Vec<_>>();
|
||||
|
||||
|
||||
{
|
||||
let op_state = js_runtime.op_state();
|
||||
let mut op_state = op_state.borrow_mut();
|
||||
@@ -527,7 +626,7 @@ pub async fn eval_fetch_timeout(
|
||||
let r = runtime.block_on(future)?;
|
||||
// tracing::info!("total: {:?}", instant.elapsed());
|
||||
|
||||
(r as anyhow::Result<Value>).map(|x| (x, js_runtime.op_state().borrow().borrow::<LogString>().s.clone()))
|
||||
(r as anyhow::Result<Box<RawValue>>).map(|x| (x, js_runtime.op_state().borrow().borrow::<LogString>().s.clone()))
|
||||
}),
|
||||
)
|
||||
.await
|
||||
@@ -541,7 +640,7 @@ pub async fn eval_fetch_timeout(
|
||||
})??
|
||||
}
|
||||
|
||||
async fn eval_fetch(js_runtime: &mut JsRuntime, expr: &str) -> anyhow::Result<serde_json::Value> {
|
||||
async fn eval_fetch(js_runtime: &mut JsRuntime, expr: &str) -> anyhow::Result<Box<RawValue>> {
|
||||
let _ = js_runtime
|
||||
.load_side_module(
|
||||
&deno_core::resolve_url("file:///eval.ts")?,
|
||||
@@ -552,8 +651,8 @@ async fn eval_fetch(js_runtime: &mut JsRuntime, expr: &str) -> anyhow::Result<se
|
||||
let global = js_runtime.execute_script(
|
||||
"<anon>",
|
||||
r#"
|
||||
let args = Deno.core.ops.op_get_static_args()
|
||||
import("file:///eval.ts").then((module) => module.main(...args))
|
||||
let args = Deno.core.ops.op_get_static_args().map(JSON.parse)
|
||||
import("file:///eval.ts").then((module) => module.main(...args)).then(JSON.stringify)
|
||||
"#
|
||||
.to_string()
|
||||
.into(),
|
||||
@@ -564,12 +663,13 @@ import("file:///eval.ts").then((module) => module.main(...args))
|
||||
let local = v8::Local::new(scope, global);
|
||||
// Deserialize a `v8` object into a Rust type using `serde_v8`,
|
||||
// in this case deserialize to a JSON `Value`.
|
||||
Ok(serde_v8::from_v8::<serde_json::Value>(scope, local)?)
|
||||
let r = serde_v8::from_v8::<String>(scope, local)?;
|
||||
Ok(unsafe_raw(r))
|
||||
}
|
||||
|
||||
#[op]
|
||||
fn op_get_static_args(op_state: Rc<RefCell<OpState>>) -> Vec<serde_json::Value> {
|
||||
return op_state.borrow().borrow::<MainArgs>().args.clone();
|
||||
fn op_get_static_args(op_state: Rc<RefCell<OpState>>) -> Vec<Option<String>> {
|
||||
return op_state.borrow().borrow::<MainArgs>().args.iter().map(|x| x.as_ref().map(|y| y.get().to_string())).collect_vec();
|
||||
}
|
||||
|
||||
#[op]
|
||||
@@ -585,21 +685,52 @@ fn op_log(op_state: Rc<RefCell<OpState>>, args: Vec<String>) {
|
||||
mod tests {
|
||||
|
||||
use serde_json::json;
|
||||
use windmill_common::worker::to_raw_value;
|
||||
|
||||
// Note this useful idiom: importing names from outer (for mod tests) scope.
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_eval() -> anyhow::Result<()> {
|
||||
let env = vec![
|
||||
("params".to_string(), json!({"test": 2})),
|
||||
("value".to_string(), json!({"test": 2})),
|
||||
];
|
||||
let mut env = HashMap::new();
|
||||
env.insert(
|
||||
"params".to_string(),
|
||||
Arc::new(to_raw_value(&json!({"test": 2}))),
|
||||
);
|
||||
env.insert(
|
||||
"value".to_string(),
|
||||
Arc::new(to_raw_value(&json!({"test": 2}))),
|
||||
);
|
||||
|
||||
let code = "value.test + params.test";
|
||||
|
||||
let mut runtime = JsRuntime::new(RuntimeOptions::default());
|
||||
let res = eval(&mut runtime, code, env, None, false).await?;
|
||||
assert_eq!(res, json!(4));
|
||||
let ops = vec![op_get_context::DECL];
|
||||
|
||||
let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() };
|
||||
let exts = vec![ext];
|
||||
|
||||
let options = RuntimeOptions {
|
||||
extensions: exts,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
let mut runtime = JsRuntime::new(options);
|
||||
{
|
||||
let op_state = runtime.op_state();
|
||||
let mut op_state = op_state.borrow_mut();
|
||||
op_state.put(TransformContext { flow_input: None, envs: env.clone() })
|
||||
}
|
||||
|
||||
let res = eval(
|
||||
&mut runtime,
|
||||
code,
|
||||
vec!["params".to_string(), "value".to_string()],
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(res.get(), "4");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -612,20 +743,33 @@ multiline template`";
|
||||
|
||||
let mut runtime = JsRuntime::new(RuntimeOptions::default());
|
||||
let res = eval(&mut runtime, code, env, None, false).await?;
|
||||
assert_eq!(res, json!("my 5\nmultiline template"));
|
||||
assert_eq!(res.get(), "\"my 5\\nmultiline template\"");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_eval_timeout() -> anyhow::Result<()> {
|
||||
let env = vec![
|
||||
("params".to_string(), json!({"test": 2})),
|
||||
("value".to_string(), json!({"test": 2})),
|
||||
];
|
||||
let mut env = HashMap::new();
|
||||
env.insert(
|
||||
"params".to_string(),
|
||||
Arc::new(to_raw_value(&json!({"test": 2}))),
|
||||
);
|
||||
env.insert(
|
||||
"value".to_string(),
|
||||
Arc::new(to_raw_value(&json!({"test": 2}))),
|
||||
);
|
||||
|
||||
let code = r#"params.test"#;
|
||||
|
||||
let res = eval_timeout(code.to_string(), env, None, None).await?;
|
||||
assert_eq!(res, json!(2));
|
||||
let mut js_runtime = JsRuntime::new(RuntimeOptions::default());
|
||||
{
|
||||
let op_state = js_runtime.op_state();
|
||||
let mut op_state = op_state.borrow_mut();
|
||||
op_state.put(TransformContext { flow_input: None, envs: env.clone() })
|
||||
}
|
||||
|
||||
let res = eval_timeout(code.to_string(), env, None, None, None).await?;
|
||||
assert_eq!(res.get(), "2");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -633,9 +777,8 @@ multiline template`";
|
||||
async fn test_eval_fetch_timeout() -> anyhow::Result<()> {
|
||||
let code = r#"export async function main() { return "" }"#;
|
||||
|
||||
let res =
|
||||
eval_fetch_timeout(code.to_string(), code.to_string(), serde_json::Map::new()).await?;
|
||||
assert_eq!(res.0, "".to_string());
|
||||
let res = eval_fetch_timeout(code.to_string(), code.to_string(), None).await?;
|
||||
assert_eq!(res.0.get(), "\"\"");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,14 +3,15 @@ use mysql_async::{
|
||||
consts::ColumnType, prelude::*, FromValueError, OptsBuilder, Params, Row, SslOpts,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
use serde_json::{json, value::RawValue, Value};
|
||||
use sqlx::types::Json;
|
||||
use windmill_common::{
|
||||
error::{to_anyhow, Error},
|
||||
jobs::QueuedJob,
|
||||
};
|
||||
use windmill_parser_sql::parse_mysql_sig;
|
||||
|
||||
use crate::{common::transform_json_value, AuthedClient};
|
||||
use crate::{common::build_args_map, AuthedClientBackgroundTask};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct MysqlDatabase {
|
||||
@@ -23,23 +24,24 @@ struct MysqlDatabase {
|
||||
}
|
||||
|
||||
pub async fn do_mysql(
|
||||
job: QueuedJob,
|
||||
client: &AuthedClient,
|
||||
job: &QueuedJob,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
query: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
) -> windmill_common::error::Result<serde_json::Value> {
|
||||
let args = if let Some(args) = &job.args {
|
||||
Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
|
||||
) -> windmill_common::error::Result<Box<RawValue>> {
|
||||
let args = build_args_map(job, client, db).await?.map(Json);
|
||||
let job_args = if args.is_some() {
|
||||
args.as_ref()
|
||||
} else {
|
||||
None
|
||||
job.args.as_ref()
|
||||
};
|
||||
|
||||
let mysql_args: serde_json::Value = serde_json::from_value(args.unwrap_or_else(|| json!({})))
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
let database = serde_json::from_value::<MysqlDatabase>(
|
||||
mysql_args.get("database").unwrap_or(&json!({})).clone(),
|
||||
)
|
||||
.map_err(|e: serde_json::Error| Error::ExecutionErr(e.to_string()))?;
|
||||
let database = if let Some(db) = job_args.and_then(|x| x.get("database")) {
|
||||
serde_json::from_str::<MysqlDatabase>(db.get())
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?
|
||||
} else {
|
||||
return Err(Error::BadRequest("Missing database argument".to_string()));
|
||||
};
|
||||
|
||||
let opts = OptsBuilder::default()
|
||||
.db_name(Some(database.database))
|
||||
@@ -61,13 +63,6 @@ pub async fn do_mysql(
|
||||
let pool = mysql_async::Pool::new(opts);
|
||||
let mut conn = pool.get_conn().await.map_err(to_anyhow)?;
|
||||
|
||||
let args = &job
|
||||
.args
|
||||
.clone()
|
||||
.unwrap_or_else(|| json!({}))
|
||||
.as_object()
|
||||
.map(|x| x.to_owned())
|
||||
.unwrap_or_else(|| json!({}).as_object().unwrap().to_owned());
|
||||
let mut statement_values: Vec<mysql_async::Value> = vec![];
|
||||
|
||||
let sig = parse_mysql_sig(&query)
|
||||
@@ -76,9 +71,18 @@ pub async fn do_mysql(
|
||||
|
||||
for arg in &sig {
|
||||
let arg_t = arg.otyp.clone().unwrap_or_else(|| "text".to_string());
|
||||
let mysql_v = match args.get(arg.name.as_str()).unwrap_or_else(|| &json!(null)) {
|
||||
let mysql_v = match job
|
||||
.args
|
||||
.as_ref()
|
||||
.and_then(|x| {
|
||||
x.get(arg.name.as_str())
|
||||
.map(|x| serde_json::from_str::<serde_json::Value>(x.get()).ok())
|
||||
})
|
||||
.flatten()
|
||||
.unwrap_or_else(|| json!(null))
|
||||
{
|
||||
Value::Null => mysql_async::Value::NULL,
|
||||
Value::Bool(b) => mysql_async::Value::Int(if *b { 1 } else { 0 }),
|
||||
Value::Bool(b) => mysql_async::Value::Int(if b { 1 } else { 0 }),
|
||||
Value::String(s) => mysql_async::Value::Bytes(s.as_bytes().to_vec()),
|
||||
Value::Number(n)
|
||||
if n.is_i64() && (arg_t == "int" || arg_t == "integer" || arg_t == "smallint") =>
|
||||
@@ -115,7 +119,7 @@ pub async fn do_mysql(
|
||||
pool.disconnect().await.map_err(to_anyhow)?;
|
||||
|
||||
// And then check that we got back the same string we sent over.
|
||||
return Ok(json!(rows));
|
||||
return Ok(windmill_common::worker::to_raw_value(&json!(rows)));
|
||||
}
|
||||
|
||||
fn convert_row_to_value(row: Row) -> serde_json::Value {
|
||||
|
||||
@@ -5,6 +5,7 @@ use native_tls::{Certificate, TlsConnector};
|
||||
use postgres_native_tls::MakeTlsConnector;
|
||||
use rust_decimal::{prelude::FromPrimitive, Decimal};
|
||||
use serde::Deserialize;
|
||||
use serde_json::value::RawValue;
|
||||
use serde_json::Map;
|
||||
use serde_json::{json, Value};
|
||||
use tokio_postgres::types::IsNull;
|
||||
@@ -18,11 +19,12 @@ use tokio_postgres::{
|
||||
};
|
||||
use uuid::Uuid;
|
||||
use windmill_common::error::{self, Error};
|
||||
use windmill_common::worker::to_raw_value;
|
||||
use windmill_common::{error::to_anyhow, jobs::QueuedJob};
|
||||
use windmill_parser_sql::parse_pgsql_sig;
|
||||
|
||||
use crate::common::transform_json_value;
|
||||
use crate::AuthedClient;
|
||||
use crate::common::build_args_values;
|
||||
use crate::AuthedClientBackgroundTask;
|
||||
use bytes::BytesMut;
|
||||
use urlencoding::encode;
|
||||
|
||||
@@ -38,22 +40,19 @@ struct PgDatabase {
|
||||
}
|
||||
|
||||
pub async fn do_postgresql(
|
||||
job: QueuedJob,
|
||||
client: &AuthedClient,
|
||||
job: &QueuedJob,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
query: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
) -> error::Result<serde_json::Value> {
|
||||
let args = if let Some(args) = &job.args {
|
||||
Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
) -> error::Result<Box<RawValue>> {
|
||||
let pg_args = build_args_values(job, client, db).await?;
|
||||
|
||||
let pg_args: serde_json::Value = serde_json::from_value(args.unwrap_or_else(|| json!({})))
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
let database =
|
||||
serde_json::from_value::<PgDatabase>(pg_args.get("database").unwrap_or(&json!({})).clone())
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
let database = if let Some(db) = pg_args.get("database") {
|
||||
serde_json::from_value::<PgDatabase>(db.clone())
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?
|
||||
} else {
|
||||
return Err(Error::BadRequest("Missing database argument".to_string()));
|
||||
};
|
||||
let sslmode = database.sslmode.unwrap_or("prefer".to_string());
|
||||
let database_string = format!(
|
||||
"postgres://{user}:{password}@{host}:{port}/{dbname}?sslmode={sslmode}",
|
||||
@@ -107,13 +106,6 @@ pub async fn do_postgresql(
|
||||
(client, handle)
|
||||
};
|
||||
|
||||
let args = &job
|
||||
.args
|
||||
.clone()
|
||||
.unwrap_or_else(|| json!({}))
|
||||
.as_object()
|
||||
.map(|x| x.to_owned())
|
||||
.unwrap_or_else(|| json!({}).as_object().unwrap().to_owned());
|
||||
let mut statement_values: Vec<serde_json::Value> = vec![];
|
||||
|
||||
let sig = parse_pgsql_sig(&query)
|
||||
@@ -121,7 +113,12 @@ pub async fn do_postgresql(
|
||||
.args;
|
||||
|
||||
for arg in &sig {
|
||||
statement_values.push(args.get(&arg.name).unwrap_or(&json!(null)).clone());
|
||||
statement_values.push(
|
||||
pg_args
|
||||
.get(&arg.name)
|
||||
.map(|x| x.to_owned())
|
||||
.unwrap_or_else(|| serde_json::Value::Null),
|
||||
);
|
||||
}
|
||||
|
||||
let query_params = statement_values
|
||||
@@ -152,7 +149,7 @@ pub async fn do_postgresql(
|
||||
|
||||
handle.abort();
|
||||
// And then check that we got back the same string we sent over.
|
||||
return Ok(result);
|
||||
return Ok(to_raw_value(&result));
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
||||
@@ -2,6 +2,7 @@ use std::{collections::HashMap, process::Stdio};
|
||||
|
||||
use itertools::Itertools;
|
||||
use regex::Regex;
|
||||
use serde_json::value::RawValue;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::{
|
||||
fs::{metadata, DirBuilder, File},
|
||||
@@ -64,6 +65,7 @@ pub async fn pip_compile(
|
||||
job_id: &Uuid,
|
||||
requirements: &str,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job_dir: &str,
|
||||
db: &Pool<Postgres>,
|
||||
worker_name: &str,
|
||||
@@ -128,6 +130,7 @@ pub async fn pip_compile(
|
||||
job_id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child_process,
|
||||
false,
|
||||
worker_name,
|
||||
@@ -164,13 +167,14 @@ pub async fn handle_python_job(
|
||||
worker_name: &str,
|
||||
job: &QueuedJob,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
inner_content: &String,
|
||||
shared_mount: &str,
|
||||
base_internal_url: &str,
|
||||
envs: HashMap<String, String>,
|
||||
) -> windmill_common::error::Result<serde_json::Value> {
|
||||
) -> windmill_common::error::Result<Box<RawValue>> {
|
||||
create_dependencies_dir(job_dir).await;
|
||||
|
||||
let mut additional_python_paths: Vec<String> = WORKER_CONFIG
|
||||
@@ -199,6 +203,7 @@ pub async fn handle_python_job(
|
||||
&job.id,
|
||||
&requirements,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -221,6 +226,7 @@ pub async fn handle_python_job(
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
logs,
|
||||
mem_peak,
|
||||
db,
|
||||
worker_name,
|
||||
job_dir,
|
||||
@@ -284,7 +290,6 @@ pub async fn handle_python_job(
|
||||
})
|
||||
.collect::<Vec<String>>()
|
||||
.join("");
|
||||
let client = client.get_authed().await;
|
||||
create_args_and_out_file(&client, job, job_dir, db).await?;
|
||||
|
||||
let import_loader = if relative_imports {
|
||||
@@ -385,6 +390,7 @@ except Exception as e:
|
||||
);
|
||||
write_file(job_dir, "wrapper.py", &wrapper_content).await?;
|
||||
|
||||
let client = client.get_authed().await;
|
||||
let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
|
||||
let additional_python_paths_folders = additional_python_paths.iter().join(":");
|
||||
if !*DISABLE_NSJAIL {
|
||||
@@ -471,6 +477,7 @@ mount {{
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child,
|
||||
!*DISABLE_NSJAIL,
|
||||
worker_name,
|
||||
@@ -488,6 +495,7 @@ pub async fn handle_python_reqs(
|
||||
job_id: &Uuid,
|
||||
w_id: &str,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
worker_name: &str,
|
||||
job_dir: &str,
|
||||
@@ -637,6 +645,7 @@ pub async fn handle_python_reqs(
|
||||
&job_id,
|
||||
db,
|
||||
logs,
|
||||
mem_peak,
|
||||
child,
|
||||
false,
|
||||
worker_name,
|
||||
|
||||
@@ -2,17 +2,17 @@ use base64::{engine, Engine as _};
|
||||
use core::fmt::Write;
|
||||
use jsonwebtoken::{encode, Algorithm, EncodingKey, Header};
|
||||
use pem;
|
||||
use serde_json::{json, Value};
|
||||
use serde_json::{json, value::RawValue, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use windmill_common::error::Error;
|
||||
use windmill_common::jobs::QueuedJob;
|
||||
use windmill_common::{error::Error, worker::to_raw_value};
|
||||
use windmill_parser_sql::parse_snowflake_sig;
|
||||
use windmill_queue::HTTP_CLIENT;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{common::transform_json_value, AuthedClient};
|
||||
use crate::{common::build_args_values, AuthedClientBackgroundTask};
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Claims {
|
||||
@@ -61,19 +61,12 @@ struct SnowflakeError {
|
||||
}
|
||||
|
||||
pub async fn do_snowflake(
|
||||
job: QueuedJob,
|
||||
client: &AuthedClient,
|
||||
job: &QueuedJob,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
query: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
) -> windmill_common::error::Result<serde_json::Value> {
|
||||
let args = if let Some(args) = &job.args {
|
||||
Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let snowflake_args: Value = serde_json::from_value(args.unwrap_or_else(|| json!({})))
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
) -> windmill_common::error::Result<Box<RawValue>> {
|
||||
let snowflake_args = build_args_values(job, client, db).await?;
|
||||
|
||||
let database = serde_json::from_value::<SnowflakeDatabase>(
|
||||
snowflake_args.get("database").unwrap_or(&json!({})).clone(),
|
||||
@@ -107,14 +100,6 @@ pub async fn do_snowflake(
|
||||
let token = encode(&Header::new(Algorithm::RS256), &claims, &private_key)
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
|
||||
let args = &job
|
||||
.args
|
||||
.clone()
|
||||
.unwrap_or_else(|| json!({}))
|
||||
.as_object()
|
||||
.map(|x| x.to_owned())
|
||||
.unwrap_or_else(|| json!({}).as_object().unwrap().to_owned());
|
||||
|
||||
let mut bindings = serde_json::Map::new();
|
||||
let sig = parse_snowflake_sig(&query)
|
||||
.map_err(|x| Error::ExecutionErr(x.to_string()))?
|
||||
@@ -123,7 +108,7 @@ pub async fn do_snowflake(
|
||||
let mut i = 1;
|
||||
for arg in &sig {
|
||||
let arg_t = arg.otyp.clone().unwrap_or_else(|| "string".to_string());
|
||||
let arg_v = args.get(&arg.name).cloned().unwrap_or(json!(""));
|
||||
let arg_v = snowflake_args.get(&arg.name).cloned().unwrap_or(json!(""));
|
||||
let snowflake_v = convert_typ_val(arg_t, arg_v);
|
||||
|
||||
bindings.insert(i.to_string(), snowflake_v);
|
||||
@@ -187,20 +172,24 @@ pub async fn do_snowflake(
|
||||
));
|
||||
}
|
||||
|
||||
let rows = result
|
||||
.data
|
||||
.iter()
|
||||
.map(|row| {
|
||||
let mut row_map = serde_json::Map::new();
|
||||
row.iter()
|
||||
.zip(result.resultSetMetaData.rowType.iter())
|
||||
.for_each(|(val, row_type)| {
|
||||
row_map
|
||||
.insert(row_type.name.clone(), parse_val(&val, &row_type.r#type));
|
||||
});
|
||||
Value::from(row_map)
|
||||
})
|
||||
.collect();
|
||||
let rows = to_raw_value(
|
||||
&result
|
||||
.data
|
||||
.iter()
|
||||
.map(|row| {
|
||||
let mut row_map = serde_json::Map::new();
|
||||
row.iter()
|
||||
.zip(result.resultSetMetaData.rowType.iter())
|
||||
.for_each(|(val, row_type)| {
|
||||
row_map.insert(
|
||||
row_type.name.clone(),
|
||||
parse_val(&val, &row_type.r#type),
|
||||
);
|
||||
});
|
||||
Value::from(row_map)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
@@ -9,11 +9,10 @@
|
||||
use anyhow::Result;
|
||||
use const_format::concatcp;
|
||||
use itertools::Itertools;
|
||||
use once_cell::sync::OnceCell;
|
||||
use prometheus::core::{AtomicU64, GenericCounter};
|
||||
#[cfg(feature = "benchmark")]
|
||||
use serde::Serialize;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use reqwest::Response;
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use sqlx::{types::Json, Pool, Postgres};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
@@ -22,7 +21,6 @@ use std::{
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use windmill_api_client::Client;
|
||||
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{
|
||||
@@ -32,14 +30,15 @@ use windmill_common::{
|
||||
scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
|
||||
users::SUPERADMIN_SECRET_EMAIL,
|
||||
utils::{rd_string, StripPath},
|
||||
worker::{update_ping, CLOUD_HOSTED, WORKER_CONFIG},
|
||||
worker::{to_raw_value, to_raw_value_owned, update_ping, CLOUD_HOSTED, WORKER_CONFIG},
|
||||
DB, IS_READY, METRICS_ENABLED,
|
||||
};
|
||||
use windmill_queue::{
|
||||
canceled_job_to_result, get_queued_job, pull, push, PushIsolationLevel, HTTP_CLIENT,
|
||||
canceled_job_to_result, empty_args, get_queued_job, pull, push, PushArgs, PushIsolationLevel,
|
||||
WrappedError, HTTP_CLIENT,
|
||||
};
|
||||
|
||||
use serde_json::{json, Value};
|
||||
use serde_json::{json, value::RawValue, Value};
|
||||
|
||||
use tokio::{
|
||||
fs::{symlink, DirBuilder},
|
||||
@@ -71,7 +70,7 @@ use windmill_queue::{add_completed_job, add_completed_job_error};
|
||||
use crate::{
|
||||
bash_executor::{handle_bash_job, handle_powershell_job, ANSI_ESCAPE_RE},
|
||||
bun_executor::{gen_lockfile, get_trusted_deps, handle_bun_job},
|
||||
common::{hash_args, read_result, save_in_cache, transform_json_value, write_file},
|
||||
common::{build_args_map, hash_args, read_result, save_in_cache, write_file},
|
||||
deno_executor::{generate_deno_lock, handle_deno_job},
|
||||
go_executor::{handle_go_job, install_go_dependencies},
|
||||
graphql_executor::do_graphql,
|
||||
@@ -98,16 +97,17 @@ pub async fn create_token_for_owner_in_bg(
|
||||
if job.workspace_id != "" {
|
||||
let mut locked = rw_lock.clone().write_owned().await;
|
||||
let db = db.clone();
|
||||
let job = job.clone();
|
||||
let w_id = job.workspace_id.clone();
|
||||
let owner = job.permissioned_as.clone();
|
||||
let email = job.email.clone();
|
||||
tokio::spawn(async move {
|
||||
let job = job.clone();
|
||||
let token = create_token_for_owner(
|
||||
&db.clone(),
|
||||
&job.workspace_id,
|
||||
&job.permissioned_as,
|
||||
&w_id,
|
||||
&owner,
|
||||
"ephemeral-script",
|
||||
*SCRIPT_TOKEN_EXPIRY,
|
||||
&job.email,
|
||||
&email,
|
||||
)
|
||||
.await
|
||||
.expect("could not create job token");
|
||||
@@ -293,7 +293,6 @@ pub struct AuthedClientBackgroundTask {
|
||||
pub base_internal_url: String,
|
||||
pub workspace: String,
|
||||
pub token: Arc<RwLock<String>>,
|
||||
pub client: OnceCell<Client>,
|
||||
}
|
||||
|
||||
impl AuthedClientBackgroundTask {
|
||||
@@ -302,7 +301,6 @@ impl AuthedClientBackgroundTask {
|
||||
base_internal_url: self.base_internal_url.clone(),
|
||||
workspace: self.workspace.clone(),
|
||||
token: self.get_token().await,
|
||||
client: self.client.clone(),
|
||||
};
|
||||
}
|
||||
pub async fn get_token(&self) -> String {
|
||||
@@ -314,14 +312,110 @@ pub struct AuthedClient {
|
||||
pub base_internal_url: String,
|
||||
pub workspace: String,
|
||||
pub token: String,
|
||||
pub client: OnceCell<Client>,
|
||||
}
|
||||
|
||||
impl AuthedClient {
|
||||
pub fn get_client(&self) -> &Client {
|
||||
return self.client.get_or_init(|| {
|
||||
windmill_api_client::create_client(&self.base_internal_url, self.token.clone())
|
||||
});
|
||||
pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result<Response> {
|
||||
Ok(HTTP_CLIENT
|
||||
.get(url)
|
||||
.query(&query)
|
||||
.header(
|
||||
reqwest::header::ACCEPT,
|
||||
reqwest::header::HeaderValue::from_static("application/json"),
|
||||
)
|
||||
.header(
|
||||
reqwest::header::AUTHORIZATION,
|
||||
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))?,
|
||||
)
|
||||
.send()
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub async fn get_resource_value<T: DeserializeOwned>(&self, path: &str) -> anyhow::Result<T> {
|
||||
let url = format!(
|
||||
"{}/api/w/{}/resources/get_value/{}",
|
||||
self.base_internal_url, self.workspace, path
|
||||
);
|
||||
let response = self.get(&url, vec![]).await?;
|
||||
match response.status().as_u16() {
|
||||
200u16 => Ok(response.json::<T>().await?),
|
||||
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_variable_value(&self, path: &str) -> anyhow::Result<String> {
|
||||
let url = format!(
|
||||
"{}/api/w/{}/variables/get_value/{}",
|
||||
self.base_internal_url, self.workspace, path
|
||||
);
|
||||
let response = self.get(&url, vec![]).await?;
|
||||
match response.status().as_u16() {
|
||||
200u16 => Ok(response.json::<String>().await?),
|
||||
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_resource_value_interpolated<T: DeserializeOwned>(
|
||||
&self,
|
||||
path: &str,
|
||||
job_id: Option<String>,
|
||||
) -> anyhow::Result<T> {
|
||||
let url = format!(
|
||||
"{}/api/w/{}/resources/get_value_interpolated/{}",
|
||||
self.base_internal_url, self.workspace, path
|
||||
);
|
||||
let mut query = Vec::with_capacity(1usize);
|
||||
if let Some(v) = &job_id {
|
||||
query.push(("job_id", v.to_string()));
|
||||
}
|
||||
let response = self.get(&url, query).await?;
|
||||
match response.status().as_u16() {
|
||||
200u16 => Ok(response.json::<T>().await?),
|
||||
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_completed_job_result<T: DeserializeOwned>(
|
||||
&self,
|
||||
path: &str,
|
||||
json_path: Option<String>,
|
||||
) -> anyhow::Result<T> {
|
||||
let url = format!(
|
||||
"{}/api/w/{}/jobs_u/completed/get_result/{}",
|
||||
self.base_internal_url, self.workspace, path
|
||||
);
|
||||
let query = if let Some(json_path) = json_path {
|
||||
vec![("json_path", json_path)]
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
let response = self.get(&url, query).await?;
|
||||
match response.status().as_u16() {
|
||||
200u16 => Ok(response.json::<T>().await?),
|
||||
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_result_by_id<T: DeserializeOwned>(
|
||||
&self,
|
||||
flow_job_id: &str,
|
||||
node_id: &str,
|
||||
json_path: Option<String>,
|
||||
) -> anyhow::Result<T> {
|
||||
let url = format!(
|
||||
"{}/api/w/{}/jobs/result_by_id/{}/{}",
|
||||
self.base_internal_url, self.workspace, flow_job_id, node_id
|
||||
);
|
||||
let query = if let Some(json_path) = json_path {
|
||||
vec![("json_path", json_path)]
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
let response = self.get(&url, query).await?;
|
||||
match response.status().as_u16() {
|
||||
200u16 => Ok(response.json::<T>().await?),
|
||||
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -357,14 +451,12 @@ async fn handle_receive_completed_job<
|
||||
let metrics = build_language_metrics(&worker_execution_failed.clone(), &jc.job.language);
|
||||
let token = jc.token.clone();
|
||||
let workspace = jc.job.workspace_id.clone();
|
||||
let client = AuthedClient {
|
||||
base_internal_url: base_internal_url.to_string(),
|
||||
workspace,
|
||||
token,
|
||||
client: OnceCell::new(),
|
||||
};
|
||||
let client =
|
||||
AuthedClient { base_internal_url: base_internal_url.to_string(), workspace, token };
|
||||
let job = jc.job.clone();
|
||||
let mem_peak = jc.mem_peak.clone();
|
||||
if let Err(err) = process_completed_job(
|
||||
&jc,
|
||||
jc,
|
||||
&client,
|
||||
&db,
|
||||
&worker_dir,
|
||||
@@ -377,7 +469,8 @@ async fn handle_receive_completed_job<
|
||||
handle_job_error(
|
||||
&db,
|
||||
&client,
|
||||
&jc.job,
|
||||
job.as_ref(),
|
||||
mem_peak,
|
||||
err,
|
||||
metrics,
|
||||
false,
|
||||
@@ -795,7 +888,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
#[cfg(feature = "enterprise")]
|
||||
{
|
||||
let (dedicated_worker_tx, dedicated_worker_rx) =
|
||||
mpsc::channel::<QueuedJob>(MAX_BUFFERED_DEDICATED_JOBS);
|
||||
mpsc::channel::<Arc<QueuedJob>>(MAX_BUFFERED_DEDICATED_JOBS);
|
||||
let mut killpill_rx = killpill_rx.resubscribe();
|
||||
let db = db.clone();
|
||||
let worker_dir = worker_dir.clone();
|
||||
@@ -891,7 +984,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
(Some(dedicated_worker_tx), Some(handle))
|
||||
}
|
||||
} else {
|
||||
(None, None) as (Option<Sender<QueuedJob>>, Option<JoinHandle<()>>)
|
||||
(None, None) as (Option<Sender<Arc<QueuedJob>>>, Option<JoinHandle<()>>)
|
||||
};
|
||||
|
||||
#[cfg(feature = "benchmark")]
|
||||
@@ -1080,7 +1173,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
#[cfg(feature = "benchmark")]
|
||||
let send_start = Instant::now();
|
||||
|
||||
if let Err(e) = dedicated_worker_tx.send(job.clone()).await {
|
||||
if let Err(e) = dedicated_worker_tx.send(Arc::new(job)).await {
|
||||
tracing::info!("failed to send jobs to dedicated workers. Likely dedicated worker has been shut down. This is normal: {e:?}");
|
||||
}
|
||||
|
||||
@@ -1097,10 +1190,11 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
|
||||
job_completed_tx
|
||||
.send(JobCompleted {
|
||||
job,
|
||||
job: Arc::new(job),
|
||||
success: true,
|
||||
result: json!({}),
|
||||
result: empty_args(),
|
||||
logs: String::new(),
|
||||
mem_peak: 0,
|
||||
cached_res_path: None,
|
||||
token: "".to_string(),
|
||||
})
|
||||
@@ -1173,11 +1267,11 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
base_internal_url: base_internal_url.to_string(),
|
||||
token,
|
||||
workspace: job.workspace_id.to_string(),
|
||||
client: OnceCell::new(),
|
||||
};
|
||||
|
||||
let arc_job = Arc::new(job);
|
||||
if let Some(err) = handle_queued_job(
|
||||
job.clone(),
|
||||
arc_job.clone(),
|
||||
db,
|
||||
&authed_client,
|
||||
&worker_name,
|
||||
@@ -1196,7 +1290,8 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
handle_job_error(
|
||||
db,
|
||||
&authed_client.get_authed().await,
|
||||
&job,
|
||||
arc_job.as_ref(),
|
||||
0,
|
||||
err,
|
||||
metrics,
|
||||
false,
|
||||
@@ -1213,7 +1308,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
.expect("no timer found")
|
||||
.inc_by(duration);
|
||||
|
||||
if !*KEEP_JOB_DIR && !(job.is_flow() && same_worker) {
|
||||
if !*KEEP_JOB_DIR && !(arc_job.is_flow() && same_worker) {
|
||||
let _ = tokio::fs::remove_dir_all(job_dir).await;
|
||||
}
|
||||
}
|
||||
@@ -1303,7 +1398,7 @@ async fn queue_init_bash_maybe<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
concurrency_time_window_s: None,
|
||||
cache_ttl: None,
|
||||
}),
|
||||
serde_json::Map::new(),
|
||||
PushArgs::empty(),
|
||||
worker_name,
|
||||
"worker@windmill.dev",
|
||||
SUPERADMIN_SECRET_EMAIL.to_string(),
|
||||
@@ -1345,7 +1440,7 @@ async fn queue_init_bash_maybe<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
// ) -> error::Result<()> {
|
||||
|
||||
pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
JobCompleted { job, result, logs, success, cached_res_path, .. }: &JobCompleted,
|
||||
JobCompleted { job, result, logs, mem_peak, success, cached_res_path, .. }: JobCompleted,
|
||||
client: &AuthedClient,
|
||||
db: &DB,
|
||||
worker_dir: &str,
|
||||
@@ -1353,18 +1448,19 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
|
||||
same_worker_tx: Sender<Uuid>,
|
||||
rsmq: Option<R>,
|
||||
) -> windmill_common::error::Result<()> {
|
||||
if *success {
|
||||
if success {
|
||||
// println!("bef completed job{:?}", SystemTime::now());
|
||||
if let Some(cached_path) = cached_res_path {
|
||||
save_in_cache(&client, &job, cached_path.to_string(), &result).await;
|
||||
save_in_cache(db, &job, cached_path.to_string(), &result).await;
|
||||
}
|
||||
add_completed_job(
|
||||
db,
|
||||
&job,
|
||||
true,
|
||||
false,
|
||||
result,
|
||||
logs.to_string(),
|
||||
Json(&result),
|
||||
logs,
|
||||
mem_peak.to_owned(),
|
||||
rsmq.clone(),
|
||||
)
|
||||
.await?;
|
||||
@@ -1377,7 +1473,7 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
true,
|
||||
result.clone(),
|
||||
&result,
|
||||
metrics.clone(),
|
||||
false,
|
||||
same_worker_tx.clone(),
|
||||
@@ -1393,7 +1489,8 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
|
||||
db,
|
||||
&job,
|
||||
logs.to_string(),
|
||||
&result,
|
||||
mem_peak.to_owned(),
|
||||
result,
|
||||
metrics.clone(),
|
||||
rsmq.clone(),
|
||||
)
|
||||
@@ -1407,7 +1504,7 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
false,
|
||||
result,
|
||||
&serde_json::value::to_raw_value(&result).unwrap(),
|
||||
metrics,
|
||||
false,
|
||||
same_worker_tx,
|
||||
@@ -1461,6 +1558,7 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
|
||||
db: &Pool<Postgres>,
|
||||
client: &AuthedClient,
|
||||
job: &QueuedJob,
|
||||
mem_peak: i32,
|
||||
err: Error,
|
||||
metrics: Option<Metrics>,
|
||||
unrecoverable: bool,
|
||||
@@ -1479,6 +1577,7 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
|
||||
db,
|
||||
job,
|
||||
format!("Unexpected error during job execution:\n{err}"),
|
||||
mem_peak,
|
||||
&err,
|
||||
metrics.clone(),
|
||||
rsmq_2,
|
||||
@@ -1497,6 +1596,7 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
|
||||
(job.id, Uuid::nil(), Some(update_job_future))
|
||||
};
|
||||
|
||||
let wrapped_error = WrappedError { error: json!(err) };
|
||||
let updated_flow = update_flow_status_after_job_completion(
|
||||
db,
|
||||
client,
|
||||
@@ -1504,7 +1604,7 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
|
||||
&job_status_to_update,
|
||||
&job.workspace_id,
|
||||
false,
|
||||
json!({ "error": err }),
|
||||
&serde_json::value::to_raw_value(&wrapped_error).unwrap(),
|
||||
metrics.clone(),
|
||||
unrecoverable,
|
||||
same_worker_tx,
|
||||
@@ -1525,6 +1625,7 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
|
||||
db,
|
||||
&parent_job,
|
||||
format!("Unexpected error during flow job error handling:\n{err}"),
|
||||
mem_peak,
|
||||
&e,
|
||||
metrics.clone(),
|
||||
rsmq,
|
||||
@@ -1545,15 +1646,18 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
|
||||
tracing::error!(job_id = %job.id, "error handling job: {err:?} {} {} {}", job.id, job.workspace_id, job.created_by);
|
||||
}
|
||||
|
||||
fn extract_error_value(log_lines: &str, i: i32) -> serde_json::Value {
|
||||
return json!({"message": format!("ExitCode: {i}, last log lines:\n{}", ANSI_ESCAPE_RE.replace_all(log_lines.trim(), "").to_string()), "name": "ExecutionErr"});
|
||||
fn extract_error_value(log_lines: &str, i: i32) -> Box<RawValue> {
|
||||
return to_raw_value(
|
||||
&json!({"message": format!("ExitCode: {i}, last log lines:\n{}", ANSI_ESCAPE_RE.replace_all(log_lines.trim(), "").to_string()), "name": "ExecutionErr"}),
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct JobCompleted {
|
||||
pub job: QueuedJob,
|
||||
pub result: serde_json::Value,
|
||||
pub job: Arc<QueuedJob>,
|
||||
pub result: Box<RawValue>,
|
||||
pub logs: String,
|
||||
pub mem_peak: i32,
|
||||
pub success: bool,
|
||||
pub cached_res_path: Option<String>,
|
||||
pub token: String,
|
||||
@@ -1582,32 +1686,38 @@ pub async fn get_content(job: &QueuedJob, db: &Pool<Postgres>) -> Result<String,
|
||||
}
|
||||
|
||||
async fn do_nativets(
|
||||
job: QueuedJob,
|
||||
job: &QueuedJob,
|
||||
logs: String,
|
||||
client: &AuthedClient,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
code: String,
|
||||
db: &Pool<Postgres>,
|
||||
) -> windmill_common::error::Result<(serde_json::Value, String)> {
|
||||
let args = if let Some(args) = &job.args {
|
||||
Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
|
||||
) -> windmill_common::error::Result<(Box<RawValue>, String)> {
|
||||
let args = build_args_map(job, client, db).await?.map(Json);
|
||||
let job_args = if args.is_some() {
|
||||
args.as_ref()
|
||||
} else {
|
||||
None
|
||||
job.args.as_ref()
|
||||
};
|
||||
|
||||
let args = args
|
||||
.as_ref()
|
||||
.map(|x| x.clone())
|
||||
.unwrap_or_else(|| json!({}))
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone();
|
||||
let result = eval_fetch_timeout(code.clone(), transpile_ts(code)?, args).await?;
|
||||
let result = eval_fetch_timeout(code.clone(), transpile_ts(code)?, job_args).await?;
|
||||
Ok((result.0, [logs, result.1].join("\n\n")))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CachedResource {
|
||||
expire: i64,
|
||||
value: Box<RawValue>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Default)]
|
||||
pub struct PreviousResult<'a> {
|
||||
#[serde(borrow)]
|
||||
pub previous_result: Option<&'a RawValue>,
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
job: QueuedJob,
|
||||
job: Arc<QueuedJob>,
|
||||
db: &DB,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
worker_name: &str,
|
||||
@@ -1619,10 +1729,10 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
job_completed_tx: Sender<JobCompleted>,
|
||||
) -> windmill_common::error::Result<()> {
|
||||
if job.canceled {
|
||||
return Err(Error::JsonErr(canceled_job_to_result(&job)))?;
|
||||
return Err(Error::JsonErr(canceled_job_to_result(&job)));
|
||||
}
|
||||
if let Some(e) = job.pre_run_error {
|
||||
return Err(Error::ExecutionErr(e));
|
||||
if let Some(e) = &job.pre_run_error {
|
||||
return Err(Error::ExecutionErr(e.to_string()));
|
||||
}
|
||||
|
||||
let step = if job.is_flow_step {
|
||||
@@ -1641,7 +1751,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
};
|
||||
|
||||
let cached_res_path = if job.cache_ttl.is_some() {
|
||||
let args_hash = hash_args(&job.args.clone().unwrap_or_else(|| json!({})));
|
||||
let args_hash = hash_args(&job.args);
|
||||
if job.is_flow_step {
|
||||
let flow_path = sqlx::query_scalar!(
|
||||
"SELECT script_path FROM queue WHERE id = $1",
|
||||
@@ -1665,54 +1775,42 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
|
||||
if let Some(cached_res_path) = cached_res_path.clone() {
|
||||
let authed_client = client.get_authed().await;
|
||||
let client: &Client = authed_client.get_client();
|
||||
let resource = client
|
||||
.get_resource_value(&job.workspace_id, &cached_res_path)
|
||||
let resource = authed_client
|
||||
.get_resource_value::<CachedResource>(&cached_res_path)
|
||||
.await;
|
||||
|
||||
if let Ok(resource) = resource {
|
||||
let v = resource.into_inner();
|
||||
if let Some(o) = v.as_object() {
|
||||
let expire = o.get("expire");
|
||||
if expire.is_some()
|
||||
&& expire
|
||||
.unwrap()
|
||||
.as_i64()
|
||||
.map(|x| x > chrono::Utc::now().timestamp())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
let result = v
|
||||
.get("value")
|
||||
.map(|x| x.to_owned())
|
||||
.unwrap_or_else(|| json!({}));
|
||||
let logs = "Job skipped because args & path found in cache and not expired"
|
||||
.to_string();
|
||||
let expire = resource.expire;
|
||||
if expire > chrono::Utc::now().timestamp() {
|
||||
let result = resource.value;
|
||||
let logs =
|
||||
"Job skipped because args & path found in cache and not expired".to_string();
|
||||
|
||||
job_completed_tx
|
||||
.send(JobCompleted {
|
||||
job,
|
||||
result,
|
||||
logs,
|
||||
success: true,
|
||||
cached_res_path: None,
|
||||
token: authed_client.token,
|
||||
})
|
||||
.await
|
||||
.expect("send job completed");
|
||||
job_completed_tx
|
||||
.send(JobCompleted {
|
||||
job: job,
|
||||
result,
|
||||
logs,
|
||||
mem_peak: 0,
|
||||
success: true,
|
||||
cached_res_path: None,
|
||||
token: authed_client.token,
|
||||
})
|
||||
.await
|
||||
.expect("send job completed");
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
};
|
||||
match job.job_kind {
|
||||
JobKind::FlowPreview | JobKind::Flow => {
|
||||
let args = job.args.clone().unwrap_or(Value::Null);
|
||||
let args = job.get_args();
|
||||
handle_flow(
|
||||
&job,
|
||||
db,
|
||||
&client.get_authed().await,
|
||||
args,
|
||||
to_raw_value(&args),
|
||||
same_worker_tx,
|
||||
worker_dir,
|
||||
rsmq,
|
||||
@@ -1721,6 +1819,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
}
|
||||
_ => {
|
||||
let mut logs = "".to_string();
|
||||
let mut mem_peak: i32 = 0;
|
||||
// println!("handle queue {:?}", SystemTime::now());
|
||||
if let Some(log_str) = &job.logs {
|
||||
logs.push_str(&log_str);
|
||||
@@ -1745,6 +1844,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
handle_dependency_job(
|
||||
&job,
|
||||
&mut logs,
|
||||
&mut mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -1757,6 +1857,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
JobKind::FlowDependencies => handle_flow_dependency_job(
|
||||
&job,
|
||||
&mut logs,
|
||||
&mut mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -1765,10 +1866,11 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
&client.get_token().await,
|
||||
)
|
||||
.await
|
||||
.map(|()| Value::Null),
|
||||
.map(|()| serde_json::from_str("{}").unwrap()),
|
||||
JobKind::AppDependencies => handle_app_dependency_job(
|
||||
&job,
|
||||
&mut logs,
|
||||
&mut mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -1777,23 +1879,23 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
&client.get_token().await,
|
||||
)
|
||||
.await
|
||||
.map(|()| Value::Null),
|
||||
JobKind::Identity => match job.args.clone() {
|
||||
Some(Value::Object(args))
|
||||
if args.len() == 1 && args.contains_key("previous_result") =>
|
||||
{
|
||||
Ok(args.get("previous_result").unwrap().clone())
|
||||
}
|
||||
args @ _ => Ok(args.unwrap_or_else(|| Value::Null)),
|
||||
},
|
||||
.map(|()| serde_json::from_str("{}").unwrap()),
|
||||
JobKind::Identity => Ok(job
|
||||
.args
|
||||
.as_ref()
|
||||
.map(|x| x.get("previous_result"))
|
||||
.flatten()
|
||||
.map(|x| x.to_owned())
|
||||
.unwrap_or_else(|| serde_json::from_str("{}").unwrap())),
|
||||
_ => {
|
||||
handle_code_execution_job(
|
||||
&job,
|
||||
job.as_ref(),
|
||||
db,
|
||||
client,
|
||||
job_dir,
|
||||
worker_dir,
|
||||
&mut logs,
|
||||
&mut mem_peak,
|
||||
base_internal_url,
|
||||
worker_name,
|
||||
)
|
||||
@@ -1802,7 +1904,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
};
|
||||
|
||||
//it's a test job, no need to update the db
|
||||
if job.workspace_id == "" {
|
||||
if job.as_ref().workspace_id == "" {
|
||||
return Ok(());
|
||||
}
|
||||
process_result(
|
||||
@@ -1811,6 +1913,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
job_dir,
|
||||
job_completed_tx,
|
||||
logs,
|
||||
mem_peak,
|
||||
cached_res_path,
|
||||
client.get_token().await,
|
||||
)
|
||||
@@ -1821,11 +1924,12 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
}
|
||||
|
||||
async fn process_result(
|
||||
job: QueuedJob,
|
||||
result: error::Result<serde_json::Value>,
|
||||
job: Arc<QueuedJob>,
|
||||
result: error::Result<Box<RawValue>>,
|
||||
job_dir: &str,
|
||||
job_completed_tx: Sender<JobCompleted>,
|
||||
logs: String,
|
||||
mem_peak: i32,
|
||||
cached_res_path: Option<String>,
|
||||
token: String,
|
||||
) -> error::Result<()> {
|
||||
@@ -1833,9 +1937,10 @@ async fn process_result(
|
||||
Ok(r) => {
|
||||
job_completed_tx
|
||||
.send(JobCompleted {
|
||||
job,
|
||||
job: job,
|
||||
result: r,
|
||||
logs,
|
||||
mem_peak,
|
||||
success: true,
|
||||
cached_res_path,
|
||||
token: token,
|
||||
@@ -1848,7 +1953,7 @@ async fn process_result(
|
||||
Error::ExitStatus(i) => {
|
||||
let res = read_result(job_dir).await.ok();
|
||||
|
||||
if res.is_some() && res.clone().unwrap().is_object() {
|
||||
if res.is_some() {
|
||||
res.unwrap()
|
||||
} else {
|
||||
let last_10_log_lines = logs
|
||||
@@ -1866,17 +1971,18 @@ async fn process_result(
|
||||
extract_error_value(log_lines, i)
|
||||
}
|
||||
}
|
||||
err @ _ => {
|
||||
json!({"message": format!("error during execution of the script:\n{}", err), "name": "ExecutionErr"})
|
||||
}
|
||||
err @ _ => to_raw_value(
|
||||
&json!({"message": format!("error during execution of the script:\n{}", err), "name": "ExecutionErr"}),
|
||||
),
|
||||
};
|
||||
|
||||
// in the happy path and if job not a flow step, we can delegate updating the completed job in the background
|
||||
job_completed_tx
|
||||
.send(JobCompleted {
|
||||
job,
|
||||
result: error_value,
|
||||
job: job,
|
||||
result: to_raw_value(&error_value),
|
||||
logs: logs,
|
||||
mem_peak,
|
||||
success: false,
|
||||
cached_res_path,
|
||||
token: token,
|
||||
@@ -1927,9 +2033,10 @@ async fn handle_code_execution_job(
|
||||
job_dir: &str,
|
||||
worker_dir: &str,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
base_internal_url: &str,
|
||||
worker_name: &str,
|
||||
) -> error::Result<serde_json::Value> {
|
||||
) -> error::Result<Box<RawValue>> {
|
||||
let (inner_content, requirements_o, language, envs) = match job.job_kind {
|
||||
JobKind::Preview => (
|
||||
job.raw_code
|
||||
@@ -1975,9 +2082,9 @@ async fn handle_code_execution_job(
|
||||
};
|
||||
|
||||
if language == Some(ScriptLang::Postgresql) {
|
||||
return do_postgresql(job.clone(), &client.get_authed().await, &inner_content, db).await;
|
||||
return do_postgresql(job, &client, &inner_content, db).await;
|
||||
} else if language == Some(ScriptLang::Mysql) {
|
||||
return do_mysql(job.clone(), &client.get_authed().await, &inner_content, db).await;
|
||||
return do_mysql(job, &client, &inner_content, db).await;
|
||||
} else if language == Some(ScriptLang::Bigquery) {
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
{
|
||||
@@ -1988,7 +2095,7 @@ async fn handle_code_execution_job(
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
{
|
||||
return do_bigquery(job.clone(), &client.get_authed().await, &inner_content, db).await;
|
||||
return do_bigquery(job, &client, &inner_content, db).await;
|
||||
}
|
||||
} else if language == Some(ScriptLang::Snowflake) {
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
@@ -2000,10 +2107,10 @@ async fn handle_code_execution_job(
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
{
|
||||
return do_snowflake(job.clone(), &client.get_authed().await, &inner_content, db).await;
|
||||
return do_snowflake(job, &client, &inner_content, db).await;
|
||||
}
|
||||
} else if language == Some(ScriptLang::Graphql) {
|
||||
return do_graphql(job.clone(), &client.get_authed().await, &inner_content, db).await;
|
||||
return do_graphql(job, &client, &inner_content, db).await;
|
||||
} else if language == Some(ScriptLang::Nativets) {
|
||||
logs.push_str("\n--- FETCH TS EXECUTION ---\n");
|
||||
let code = format!(
|
||||
@@ -2011,14 +2118,7 @@ async fn handle_code_execution_job(
|
||||
&client.get_token().await,
|
||||
inner_content
|
||||
);
|
||||
let (result, ts_logs) = do_nativets(
|
||||
job.clone(),
|
||||
logs.clone(),
|
||||
&client.get_authed().await,
|
||||
code,
|
||||
db,
|
||||
)
|
||||
.await?;
|
||||
let (result, ts_logs) = do_nativets(job, logs.clone(), &client, code, db).await?;
|
||||
*logs = ts_logs;
|
||||
return Ok(result);
|
||||
}
|
||||
@@ -2057,7 +2157,7 @@ mount {{
|
||||
|
||||
let envs = build_envs(envs)?;
|
||||
|
||||
let result: error::Result<serde_json::Value> = match language {
|
||||
let result: error::Result<Box<RawValue>> = match language {
|
||||
None => {
|
||||
return Err(Error::ExecutionErr(
|
||||
"Require language to be not null".to_string(),
|
||||
@@ -2071,6 +2171,7 @@ mount {{
|
||||
worker_name,
|
||||
job,
|
||||
logs,
|
||||
mem_peak,
|
||||
db,
|
||||
client,
|
||||
&inner_content,
|
||||
@@ -2084,6 +2185,7 @@ mount {{
|
||||
handle_deno_job(
|
||||
requirements_o,
|
||||
logs,
|
||||
mem_peak,
|
||||
job,
|
||||
db,
|
||||
client,
|
||||
@@ -2099,6 +2201,7 @@ mount {{
|
||||
handle_bun_job(
|
||||
requirements_o,
|
||||
logs,
|
||||
mem_peak,
|
||||
job,
|
||||
db,
|
||||
client,
|
||||
@@ -2114,6 +2217,7 @@ mount {{
|
||||
Some(ScriptLang::Go) => {
|
||||
handle_go_job(
|
||||
logs,
|
||||
mem_peak,
|
||||
job,
|
||||
db,
|
||||
client,
|
||||
@@ -2130,6 +2234,7 @@ mount {{
|
||||
Some(ScriptLang::Bash) => {
|
||||
handle_bash_job(
|
||||
logs,
|
||||
mem_peak,
|
||||
job,
|
||||
db,
|
||||
client,
|
||||
@@ -2145,6 +2250,7 @@ mount {{
|
||||
Some(ScriptLang::Powershell) => {
|
||||
handle_powershell_job(
|
||||
logs,
|
||||
mem_peak,
|
||||
job,
|
||||
db,
|
||||
client,
|
||||
@@ -2177,13 +2283,14 @@ mount {{
|
||||
async fn handle_dependency_job(
|
||||
job: &QueuedJob,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job_dir: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
worker_name: &str,
|
||||
worker_dir: &str,
|
||||
base_internal_url: &str,
|
||||
token: &str,
|
||||
) -> error::Result<serde_json::Value> {
|
||||
) -> error::Result<Box<RawValue>> {
|
||||
let content = capture_dependency_job(
|
||||
&job.id,
|
||||
job.language.as_ref().map(|v| Ok(v)).unwrap_or_else(|| {
|
||||
@@ -2196,6 +2303,7 @@ async fn handle_dependency_job(
|
||||
.map(|a| a.as_str())
|
||||
.unwrap_or_else(|| "no raw code"),
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -2216,7 +2324,9 @@ async fn handle_dependency_job(
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
Ok(json!({ "success": "Successful lock file generation", "lock": content }))
|
||||
Ok(to_raw_value_owned(
|
||||
json!({ "success": "Successful lock file generation", "lock": content }),
|
||||
))
|
||||
}
|
||||
Err(error) => {
|
||||
sqlx::query!(
|
||||
@@ -2235,6 +2345,7 @@ async fn handle_dependency_job(
|
||||
async fn handle_flow_dependency_job(
|
||||
job: &QueuedJob,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job_dir: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
worker_name: &str,
|
||||
@@ -2258,6 +2369,7 @@ async fn handle_flow_dependency_job(
|
||||
flow.modules,
|
||||
job,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -2298,6 +2410,7 @@ async fn lock_modules(
|
||||
modules: Vec<FlowModule>,
|
||||
job: &QueuedJob,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job_dir: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
worker_name: &str,
|
||||
@@ -2333,6 +2446,7 @@ async fn lock_modules(
|
||||
modules,
|
||||
job,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -2354,6 +2468,7 @@ async fn lock_modules(
|
||||
b.modules,
|
||||
job,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -2374,6 +2489,7 @@ async fn lock_modules(
|
||||
b.modules,
|
||||
job,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -2389,6 +2505,7 @@ async fn lock_modules(
|
||||
default,
|
||||
job,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -2422,6 +2539,7 @@ async fn lock_modules(
|
||||
&language,
|
||||
&dependencies,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -2479,6 +2597,7 @@ async fn lock_modules_app(
|
||||
value: Value,
|
||||
job: &QueuedJob,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job_dir: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
worker_name: &str,
|
||||
@@ -2521,6 +2640,7 @@ async fn lock_modules_app(
|
||||
&language,
|
||||
&dependencies,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -2560,6 +2680,7 @@ async fn lock_modules_app(
|
||||
b,
|
||||
job,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -2581,6 +2702,7 @@ async fn lock_modules_app(
|
||||
b,
|
||||
job,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -2601,6 +2723,7 @@ async fn lock_modules_app(
|
||||
async fn handle_app_dependency_job(
|
||||
job: &QueuedJob,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job_dir: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
worker_name: &str,
|
||||
@@ -2628,6 +2751,7 @@ async fn handle_app_dependency_job(
|
||||
value,
|
||||
job,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
@@ -2665,6 +2789,7 @@ async fn capture_dependency_job(
|
||||
job_language: &ScriptLang,
|
||||
job_raw_code: &str,
|
||||
logs: &mut String,
|
||||
mem_peak: &mut i32,
|
||||
job_dir: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
worker_name: &str,
|
||||
@@ -2677,8 +2802,17 @@ async fn capture_dependency_job(
|
||||
match job_language {
|
||||
ScriptLang::Python3 => {
|
||||
create_dependencies_dir(job_dir).await;
|
||||
let req: std::result::Result<String, Error> =
|
||||
pip_compile(job_id, job_raw_code, logs, job_dir, db, worker_name, w_id).await;
|
||||
let req: std::result::Result<String, Error> = pip_compile(
|
||||
job_id,
|
||||
job_raw_code,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
w_id,
|
||||
)
|
||||
.await;
|
||||
// install the dependencies to pre-fill the cache
|
||||
if let Ok(req) = req.as_ref() {
|
||||
let r = handle_python_reqs(
|
||||
@@ -2686,6 +2820,7 @@ async fn capture_dependency_job(
|
||||
job_id,
|
||||
w_id,
|
||||
logs,
|
||||
mem_peak,
|
||||
db,
|
||||
worker_name,
|
||||
job_dir,
|
||||
@@ -2708,6 +2843,7 @@ async fn capture_dependency_job(
|
||||
job_id,
|
||||
job_raw_code,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
false,
|
||||
@@ -2723,6 +2859,7 @@ async fn capture_dependency_job(
|
||||
job_id,
|
||||
job_raw_code,
|
||||
logs,
|
||||
mem_peak,
|
||||
job_dir,
|
||||
db,
|
||||
w_id,
|
||||
@@ -2737,6 +2874,7 @@ async fn capture_dependency_job(
|
||||
let trusted_deps = get_trusted_deps(job_raw_code);
|
||||
let req = gen_lockfile(
|
||||
logs,
|
||||
mem_peak,
|
||||
job_id,
|
||||
w_id,
|
||||
db,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -478,7 +478,7 @@
|
||||
on:click={() => editor?.reloadWebsocket()}
|
||||
startIcon={{
|
||||
icon: faRotate,
|
||||
classes: !websocketAlive[lang] ? 'animate-spin' : ''
|
||||
classes: websocketAlive[lang] == false ? 'animate-spin' : ''
|
||||
}}
|
||||
title="Reload assistants"
|
||||
>
|
||||
|
||||
@@ -403,9 +403,12 @@
|
||||
<TabContent value={'oauth'}>
|
||||
<div>
|
||||
<h4 class="pb-4">SSO</h4>
|
||||
<Alert type="warning" title="Limited to 50 SSO users">
|
||||
Without EE, the number of SSO users is limited to 50. SCIM/SAML is available on EE
|
||||
</Alert>
|
||||
{#if !$enterpriseLicense}
|
||||
<Alert type="warning" title="Limited to 50 SSO users">
|
||||
Without EE, the number of SSO users is limited to 50. SCIM/SAML is available on EE
|
||||
</Alert>
|
||||
{/if}
|
||||
|
||||
<div class="py-1" />
|
||||
<Alert type="info" title="Test on a separate tab">
|
||||
The recommended workflow is to to save your oauth setting and test them directly on the
|
||||
|
||||
@@ -1,451 +1,23 @@
|
||||
<script lang="ts">
|
||||
import { Alert, Button } from '$lib/components/common'
|
||||
import Drawer from '$lib/components/common/drawer/Drawer.svelte'
|
||||
import DrawerContent from '$lib/components/common/drawer/DrawerContent.svelte'
|
||||
import CronInput from '$lib/components/CronInput.svelte'
|
||||
import Path from '$lib/components/Path.svelte'
|
||||
import Required from '$lib/components/Required.svelte'
|
||||
import SchemaForm from '$lib/components/SchemaForm.svelte'
|
||||
import ScriptPicker from '$lib/components/ScriptPicker.svelte'
|
||||
import ErrorOrRecoveryHandler from '$lib/components/ErrorOrRecoveryHandler.svelte'
|
||||
import Toggle from '$lib/components/Toggle.svelte'
|
||||
import Tooltip from '$lib/components/Tooltip.svelte'
|
||||
import { FlowService, ScheduleService, Script, ScriptService, type Flow } from '$lib/gen'
|
||||
import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores'
|
||||
import { canWrite, emptyString, formatCron, sendUserToast } from '$lib/utils'
|
||||
import { faList, faSave } from '@fortawesome/free-solid-svg-icons'
|
||||
import { createEventDispatcher } from 'svelte'
|
||||
import Section from '$lib/components/Section.svelte'
|
||||
import { tick } from 'svelte'
|
||||
import ScheduleEditorInner from './ScheduleEditorInner.svelte'
|
||||
|
||||
const slackErrorHandler = 'hub/2431/slack/schedule-error-handler-slack'
|
||||
const slackRecoveryHandler = 'hub/2430/slack/schedule-recovery-handler-slack'
|
||||
|
||||
let initialPath = ''
|
||||
let edit = true
|
||||
let schedule: string = '0 0 12 * *'
|
||||
let timezone: string = Intl.DateTimeFormat().resolvedOptions().timeZone
|
||||
|
||||
let itemKind: 'flow' | 'script' = 'script'
|
||||
let errorHandleritemKind: 'flow' | 'script' = 'script'
|
||||
let errorHandlerPath: string | undefined = undefined
|
||||
let errorHandlerCustomInitialPath: string | undefined = undefined
|
||||
let errorHandlerSelected: 'custom' | 'slack' = 'slack'
|
||||
let errorHandlerExtraArgs: Record<string, any> = {}
|
||||
let recoveryHandlerPath: string | undefined = undefined
|
||||
let recoveryHandlerCustomInitialPath: string | undefined = undefined
|
||||
let recoveryHandlerSelected: 'custom' | 'slack' = 'slack'
|
||||
let recoveryHandlerItemKind: 'flow' | 'script' = 'script'
|
||||
let recoveryHandlerExtraArgs: Record<string, any> = {}
|
||||
let failedTimes = 1
|
||||
let failedExact = false
|
||||
let recoveredTimes = 1
|
||||
|
||||
let script_path = ''
|
||||
let initialScriptPath = ''
|
||||
|
||||
export function openEdit(ePath: string, isFlow: boolean) {
|
||||
is_flow = isFlow
|
||||
initialPath = ePath
|
||||
itemKind = is_flow ? 'flow' : 'script'
|
||||
if (path == ePath) {
|
||||
loadSchedule()
|
||||
} else {
|
||||
path = ePath
|
||||
}
|
||||
edit = true
|
||||
drawer?.openDrawer()
|
||||
let open = false
|
||||
export async function openEdit(ePath: string, isFlow: boolean) {
|
||||
open = true
|
||||
await tick()
|
||||
drawer?.openEdit(ePath, isFlow)
|
||||
}
|
||||
|
||||
export function openNew(is_flow: boolean, initial_script_path?: string) {
|
||||
edit = false
|
||||
itemKind = is_flow ? 'flow' : 'script'
|
||||
initialScriptPath = initial_script_path ?? ''
|
||||
path = initialScriptPath
|
||||
initialPath = initialScriptPath
|
||||
script_path = initialScriptPath
|
||||
errorHandlerSelected = $enterpriseLicense ? 'slack' : 'custom'
|
||||
errorHandleritemKind = 'script'
|
||||
errorHandlerPath = undefined
|
||||
errorHandlerCustomInitialPath = undefined
|
||||
errorHandlerExtraArgs = {}
|
||||
recoveryHandlerSelected = $enterpriseLicense ? 'slack' : 'custom'
|
||||
recoveryHandlerPath = undefined
|
||||
recoveryHandlerCustomInitialPath = undefined
|
||||
recoveryHandlerItemKind = 'script'
|
||||
recoveryHandlerExtraArgs = {}
|
||||
timezone = Intl.DateTimeFormat().resolvedOptions().timeZone
|
||||
drawer?.openDrawer()
|
||||
export async function openNew(is_flow: boolean, initial_script_path?: string) {
|
||||
open = true
|
||||
await tick()
|
||||
drawer?.openNew(is_flow, initial_script_path)
|
||||
}
|
||||
|
||||
$: is_flow = itemKind == 'flow'
|
||||
|
||||
let runnable: Script | Flow | undefined
|
||||
let args: Record<string, any> = {}
|
||||
|
||||
let isValid = true
|
||||
|
||||
let path: string = ''
|
||||
let enabled: boolean = false
|
||||
let pathError = ''
|
||||
|
||||
let validCRON = true
|
||||
$: allowSchedule = isValid && validCRON && script_path != ''
|
||||
|
||||
$: script_path != '' && loadScript(script_path)
|
||||
|
||||
// set isValid to true when a script/flow without any properties is selected
|
||||
$: runnable?.schema &&
|
||||
runnable.schema.properties &&
|
||||
Object.keys(runnable.schema.properties).length === 0 &&
|
||||
(isValid = true)
|
||||
|
||||
const dispatch = createEventDispatcher()
|
||||
|
||||
async function loadScript(p: string | undefined): Promise<void> {
|
||||
if (p) {
|
||||
if (is_flow) {
|
||||
runnable = await FlowService.getFlowByPath({ workspace: $workspaceStore!, path: p })
|
||||
} else {
|
||||
runnable = await ScriptService.getScriptByPath({ workspace: $workspaceStore!, path: p })
|
||||
}
|
||||
} else {
|
||||
runnable = undefined
|
||||
}
|
||||
}
|
||||
|
||||
let can_write = true
|
||||
async function loadSchedule(): Promise<void> {
|
||||
try {
|
||||
const s = await ScheduleService.getSchedule({
|
||||
workspace: $workspaceStore!,
|
||||
path: initialPath
|
||||
})
|
||||
enabled = s.enabled
|
||||
schedule = s.schedule
|
||||
timezone = s.timezone
|
||||
script_path = s.script_path ?? ''
|
||||
is_flow = s.is_flow
|
||||
if (s.on_failure) {
|
||||
let splitted = s.on_failure.split('/')
|
||||
errorHandleritemKind = splitted[0] as 'flow' | 'script'
|
||||
errorHandlerPath = splitted.slice(1)?.join('/')
|
||||
errorHandlerCustomInitialPath = errorHandlerPath
|
||||
failedTimes = s.on_failure_times ?? 1
|
||||
failedExact = s.on_failure_exact ?? false
|
||||
errorHandlerExtraArgs = s.on_failure_extra_args ?? {}
|
||||
if (errorHandlerPath !== slackErrorHandler) {
|
||||
errorHandlerSelected = 'custom'
|
||||
}
|
||||
} else {
|
||||
errorHandlerPath = undefined
|
||||
errorHandleritemKind = 'script'
|
||||
}
|
||||
if (s.on_recovery) {
|
||||
let splitted = s.on_recovery.split('/')
|
||||
recoveryHandlerItemKind = splitted[0] as 'flow' | 'script'
|
||||
recoveryHandlerPath = splitted.slice(1)?.join('/')
|
||||
recoveryHandlerCustomInitialPath = recoveryHandlerPath
|
||||
recoveredTimes = s.on_recovery_times ?? 1
|
||||
recoveryHandlerExtraArgs = s.on_recovery_extra_args ?? {}
|
||||
if (recoveryHandlerPath !== slackRecoveryHandler) {
|
||||
recoveryHandlerSelected = 'custom'
|
||||
}
|
||||
} else {
|
||||
recoveryHandlerPath = undefined
|
||||
recoveryHandlerItemKind = 'script'
|
||||
}
|
||||
args = s.args ?? {}
|
||||
can_write = canWrite(s.path, s.extra_perms, $userStore)
|
||||
} catch (err) {
|
||||
sendUserToast(`Could not load schedule: ${err}`, true)
|
||||
}
|
||||
}
|
||||
|
||||
async function scheduleScript(): Promise<void> {
|
||||
if (errorHandlerSelected === 'slack' && !emptyString(errorHandlerPath)) {
|
||||
errorHandlerExtraArgs['slack'] = '$res:f/slack_bot/bot_token'
|
||||
}
|
||||
if (recoveryHandlerSelected === 'slack' && !emptyString(recoveryHandlerPath)) {
|
||||
recoveryHandlerExtraArgs['slack'] = '$res:f/slack_bot/bot_token'
|
||||
}
|
||||
if (edit) {
|
||||
await ScheduleService.updateSchedule({
|
||||
workspace: $workspaceStore!,
|
||||
path: initialPath,
|
||||
requestBody: {
|
||||
schedule: formatCron(schedule),
|
||||
timezone,
|
||||
args,
|
||||
on_failure: errorHandlerPath ? `${errorHandleritemKind}/${errorHandlerPath}` : undefined,
|
||||
on_failure_times: failedTimes,
|
||||
on_failure_exact: failedExact,
|
||||
on_failure_extra_args: errorHandlerPath ? errorHandlerExtraArgs : undefined,
|
||||
on_recovery: recoveryHandlerPath
|
||||
? `${recoveryHandlerItemKind}/${recoveryHandlerPath}`
|
||||
: undefined,
|
||||
on_recovery_times: recoveredTimes,
|
||||
on_recovery_extra_args: recoveryHandlerPath ? recoveryHandlerExtraArgs : {},
|
||||
}
|
||||
})
|
||||
sendUserToast(`Schedule ${path} updated`)
|
||||
} else {
|
||||
await ScheduleService.createSchedule({
|
||||
workspace: $workspaceStore!,
|
||||
requestBody: {
|
||||
path,
|
||||
schedule: formatCron(schedule),
|
||||
timezone,
|
||||
script_path,
|
||||
is_flow,
|
||||
args,
|
||||
enabled: true,
|
||||
on_failure: errorHandlerPath ? `${errorHandleritemKind}/${errorHandlerPath}` : undefined,
|
||||
on_failure_times: failedTimes,
|
||||
on_failure_exact: failedExact,
|
||||
on_failure_extra_args: errorHandlerPath ? errorHandlerExtraArgs : undefined,
|
||||
on_recovery: recoveryHandlerPath
|
||||
? `${recoveryHandlerItemKind}/${recoveryHandlerPath}`
|
||||
: undefined,
|
||||
on_recovery_times: recoveredTimes,
|
||||
on_recovery_extra_args: recoveryHandlerPath ? recoveryHandlerExtraArgs : {},
|
||||
}
|
||||
})
|
||||
sendUserToast(`Schedule ${path} created`)
|
||||
}
|
||||
dispatch('update')
|
||||
drawer.closeDrawer()
|
||||
}
|
||||
|
||||
$: {
|
||||
if ($workspaceStore) {
|
||||
if (edit && path != '') {
|
||||
loadSchedule()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let drawer: Drawer
|
||||
let drawer: ScheduleEditorInner
|
||||
</script>
|
||||
|
||||
<Drawer size="900px" bind:this={drawer}>
|
||||
<DrawerContent
|
||||
title={edit ? `Edit schedule ${initialPath}` : 'New schedule'}
|
||||
on:close={drawer.closeDrawer}
|
||||
>
|
||||
<svelte:fragment slot="actions">
|
||||
{#if edit}
|
||||
<div class="mr-8">
|
||||
<Button
|
||||
size="sm"
|
||||
variant="border"
|
||||
startIcon={{ icon: faList }}
|
||||
disabled={!allowSchedule || pathError != '' || emptyString(script_path)}
|
||||
href={`/runs/${script_path}`}
|
||||
>
|
||||
View Runs
|
||||
</Button>
|
||||
</div>
|
||||
<div class="mr-8 center-center -mt-2">
|
||||
<Toggle
|
||||
disabled={!can_write}
|
||||
checked={enabled}
|
||||
options={{ right: 'enable', left: 'disable' }}
|
||||
on:change={async (e) => {
|
||||
await ScheduleService.setScheduleEnabled({
|
||||
path: initialPath,
|
||||
workspace: $workspaceStore ?? '',
|
||||
requestBody: { enabled: e.detail }
|
||||
})
|
||||
sendUserToast(`${e.detail ? 'enabled' : 'disabled'} schedule ${initialPath}`)
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{/if}
|
||||
<Button
|
||||
startIcon={{ icon: faSave }}
|
||||
disabled={!allowSchedule || pathError != '' || emptyString(script_path) || (errorHandlerSelected == 'slack' && !emptyString(errorHandlerPath) && emptyString(errorHandlerExtraArgs['channel']))}
|
||||
on:click={scheduleScript}
|
||||
>
|
||||
{edit ? 'Save' : 'Schedule'}
|
||||
</Button>
|
||||
</svelte:fragment>
|
||||
|
||||
<div class="flex flex-col gap-8">
|
||||
{#if !edit}
|
||||
<Section label="Metadata">
|
||||
<Path
|
||||
checkInitialPathExistence
|
||||
bind:error={pathError}
|
||||
bind:path
|
||||
{initialPath}
|
||||
namePlaceholder="schedule"
|
||||
kind="schedule"
|
||||
/>
|
||||
</Section>
|
||||
{/if}
|
||||
<Section label="Schedule">
|
||||
<svelte:fragment slot="header">
|
||||
<Tooltip>Schedules use CRON syntax. Seconds are mandatory.</Tooltip>
|
||||
</svelte:fragment>
|
||||
<CronInput disabled={!can_write} bind:schedule bind:timezone bind:validCRON />
|
||||
</Section>
|
||||
<Section label="Runnable">
|
||||
{#if !edit}
|
||||
<p class="text-xs mb-1 text-tertiary">
|
||||
Pick a script or flow to be triggered by the schedule<Required required={true} />
|
||||
</p>
|
||||
<ScriptPicker
|
||||
disabled={initialScriptPath != '' || !can_write}
|
||||
initialPath={initialScriptPath}
|
||||
kinds={[Script.kind.SCRIPT]}
|
||||
allowFlow={true}
|
||||
bind:itemKind
|
||||
bind:scriptPath={script_path}
|
||||
/>
|
||||
{:else}
|
||||
<Alert type="info" title="Runnable path cannot be edited">
|
||||
Once a schedule is created, the runnable path cannot be changed. However, when renaming
|
||||
a script or a flow, the runnable path will automatically update itself.
|
||||
</Alert>
|
||||
<div class="my-2" />
|
||||
<ScriptPicker
|
||||
disabled
|
||||
initialPath={script_path}
|
||||
scriptPath={script_path}
|
||||
allowFlow={true}
|
||||
{itemKind}
|
||||
/>
|
||||
{/if}
|
||||
<div class="mt-6">
|
||||
{#if runnable}
|
||||
{#if runnable?.schema && runnable.schema.properties && Object.keys(runnable.schema.properties).length > 0}
|
||||
<SchemaForm disabled={!can_write} schema={runnable.schema} bind:isValid bind:args />
|
||||
{:else}
|
||||
<div class="text-xs texg-gray-700">
|
||||
This {is_flow ? 'flow' : 'script'} takes no argument
|
||||
</div>
|
||||
{/if}
|
||||
{:else}
|
||||
<div class="text-xs texg-gray-700 my-2">
|
||||
Pick a {is_flow ? 'flow' : 'script'} and fill its argument here
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</Section>
|
||||
<Section label="Error handler">
|
||||
<ErrorOrRecoveryHandler
|
||||
isEditable={can_write}
|
||||
handlersOnlyForEe={['slack']}
|
||||
showScriptHelpText={true}
|
||||
bind:handlerSelected={errorHandlerSelected}
|
||||
bind:handlerPath={errorHandlerPath}
|
||||
customInitialScriptPath={errorHandlerCustomInitialPath}
|
||||
slackHandlerScriptPath={slackErrorHandler}
|
||||
slackToggleText="Alert channel on error"
|
||||
customScriptTemplate="/scripts/add?hub=hub%2F2420%2Fwindmill%2Fschedule_error_handler_template"
|
||||
bind:customHandlerKind={errorHandleritemKind}
|
||||
bind:handlerExtraArgs={errorHandlerExtraArgs}
|
||||
>
|
||||
<svelte:fragment slot="custom-tab-tooltip">
|
||||
<Tooltip>
|
||||
<div class="flex gap-20 items-start mt-3">
|
||||
<div class="text-sm"
|
||||
>The following args will be passed to the error handler:
|
||||
<ul class="mt-1 ml-2">
|
||||
<li><b>path</b>: The path of the script or flow that failed.</li>
|
||||
<li><b>is_flow</b>: Whether the runnable is a flow.</li>
|
||||
<li><b>schedule_path</b>: The path of the schedule.</li>
|
||||
<li><b>error</b>: The error details.</li>
|
||||
<li
|
||||
><b>failed_times</b>: Minimum number of times the schedule failed before calling
|
||||
the error handler.</li
|
||||
>
|
||||
<li><b>started_at</b>: The start datetime of the latest job that failed.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</Tooltip>
|
||||
</svelte:fragment>
|
||||
</ErrorOrRecoveryHandler>
|
||||
|
||||
<div class="flex flex-row items-center justify-between">
|
||||
<div class="flex flex-row items-center mt-4 font-semibold text-sm gap-2">
|
||||
<p class="{emptyString(errorHandlerPath) ? 'text-tertiary' : ''}"
|
||||
>{#if !$enterpriseLicense}<span class="text-normal text-2xs">(ee only)</span>{/if} Triggered
|
||||
when schedule failed</p
|
||||
>
|
||||
<select class="!w-14" bind:value={failedExact} disabled={!$enterpriseLicense || emptyString(errorHandlerPath)}>
|
||||
<option value={false}>>=</option>
|
||||
<option value={true}>==</option>
|
||||
</select>
|
||||
<input
|
||||
type="number"
|
||||
class="!w-14 text-center {emptyString(errorHandlerPath) ? 'text-tertiary' : ''}"
|
||||
bind:value={failedTimes}
|
||||
disabled={!$enterpriseLicense}
|
||||
min="1"
|
||||
/>
|
||||
<p class="{emptyString(errorHandlerPath) ? 'text-tertiary' : ''}">time{failedTimes > 1 ? 's in a row' : ''}</p>
|
||||
</div>
|
||||
</div>
|
||||
</Section>
|
||||
<Section label="Recovery handler">
|
||||
<svelte:fragment slot="header">
|
||||
<div class="flex flex-row gap-2">
|
||||
{#if !$enterpriseLicense}<span class="text-normal text-2xs">(ee only)</span>{/if}
|
||||
</div>
|
||||
</svelte:fragment>
|
||||
|
||||
<ErrorOrRecoveryHandler
|
||||
isEditable={can_write && !emptyString($enterpriseLicense)}
|
||||
handlersOnlyForEe={[]}
|
||||
bind:handlerSelected={recoveryHandlerSelected}
|
||||
bind:handlerPath={recoveryHandlerPath}
|
||||
customInitialScriptPath={recoveryHandlerCustomInitialPath}
|
||||
slackHandlerScriptPath={slackRecoveryHandler}
|
||||
slackToggleText="Alert channel when error recovered"
|
||||
customScriptTemplate="/scripts/add?hub=hub%2F2421%2Fwindmill%2Fschedule_recovery_handler_template"
|
||||
bind:customHandlerKind={recoveryHandlerItemKind}
|
||||
bind:handlerExtraArgs={recoveryHandlerExtraArgs}
|
||||
>
|
||||
<svelte:fragment slot="custom-tab-tooltip">
|
||||
<Tooltip>
|
||||
<div class="flex gap-20 items-start mt-3">
|
||||
<div class=" text-sm"
|
||||
>The following args will be passed to the recovery handler:
|
||||
<ul class="mt-1 ml-2">
|
||||
<li><b>path</b>: The path of the script or flow that recovered.</li>
|
||||
<li><b>is_flow</b>: Whether the runnable is a flow.</li>
|
||||
<li><b>schedule_path</b>: The path of the schedule.</li>
|
||||
<li><b>error</b>: The error of the last job that errored</li>
|
||||
<li><b>error_started_at</b>: The start datetime of the last job that errored</li>
|
||||
<li
|
||||
><b>success_times</b>: The number of times the schedule succeeded before calling
|
||||
the recovery handler.</li
|
||||
>
|
||||
<li><b>success_result</b>: The result of the latest successful job</li>
|
||||
<li><b>success_started_at</b>: The start datetime of the latest successful job</li
|
||||
>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</Tooltip>
|
||||
</svelte:fragment>
|
||||
</ErrorOrRecoveryHandler>
|
||||
|
||||
<div class="flex flex-row items-center justify-between">
|
||||
<div class="flex flex-row items-center mt-5 font-semibold text-sm {emptyString(recoveryHandlerPath) ? 'text-tertiary' : ''}">
|
||||
<p>Triggered when schedule recovered</p>
|
||||
<input
|
||||
type="number"
|
||||
class="!w-14 mx-2 text-center"
|
||||
bind:value={recoveredTimes}
|
||||
min="1"
|
||||
/>
|
||||
<p>time{recoveredTimes > 1 ? 's in a row' : ''}</p>
|
||||
</div>
|
||||
</div>
|
||||
</Section>
|
||||
</div>
|
||||
</DrawerContent>
|
||||
</Drawer>
|
||||
{#if open}
|
||||
<ScheduleEditorInner on:update bind:this={drawer} />
|
||||
{/if}
|
||||
|
||||
@@ -0,0 +1,470 @@
|
||||
<script lang="ts">
|
||||
import { Alert, Button } from '$lib/components/common'
|
||||
import Drawer from '$lib/components/common/drawer/Drawer.svelte'
|
||||
import DrawerContent from '$lib/components/common/drawer/DrawerContent.svelte'
|
||||
import CronInput from '$lib/components/CronInput.svelte'
|
||||
import Path from '$lib/components/Path.svelte'
|
||||
import Required from '$lib/components/Required.svelte'
|
||||
import SchemaForm from '$lib/components/SchemaForm.svelte'
|
||||
import ScriptPicker from '$lib/components/ScriptPicker.svelte'
|
||||
import ErrorOrRecoveryHandler from '$lib/components/ErrorOrRecoveryHandler.svelte'
|
||||
import Toggle from '$lib/components/Toggle.svelte'
|
||||
import Tooltip from '$lib/components/Tooltip.svelte'
|
||||
import { FlowService, ScheduleService, Script, ScriptService, type Flow } from '$lib/gen'
|
||||
import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores'
|
||||
import { canWrite, emptyString, formatCron, sendUserToast } from '$lib/utils'
|
||||
import { faList, faSave } from '@fortawesome/free-solid-svg-icons'
|
||||
import { createEventDispatcher } from 'svelte'
|
||||
import Section from '$lib/components/Section.svelte'
|
||||
|
||||
const slackErrorHandler = 'hub/2431/slack/schedule-error-handler-slack'
|
||||
const slackRecoveryHandler = 'hub/2430/slack/schedule-recovery-handler-slack'
|
||||
|
||||
let initialPath = ''
|
||||
let edit = true
|
||||
let schedule: string = '0 0 12 * *'
|
||||
let timezone: string = Intl.DateTimeFormat().resolvedOptions().timeZone
|
||||
|
||||
let itemKind: 'flow' | 'script' = 'script'
|
||||
let errorHandleritemKind: 'flow' | 'script' = 'script'
|
||||
let errorHandlerPath: string | undefined = undefined
|
||||
let errorHandlerCustomInitialPath: string | undefined = undefined
|
||||
let errorHandlerSelected: 'custom' | 'slack' = 'slack'
|
||||
let errorHandlerExtraArgs: Record<string, any> = {}
|
||||
let recoveryHandlerPath: string | undefined = undefined
|
||||
let recoveryHandlerCustomInitialPath: string | undefined = undefined
|
||||
let recoveryHandlerSelected: 'custom' | 'slack' = 'slack'
|
||||
let recoveryHandlerItemKind: 'flow' | 'script' = 'script'
|
||||
let recoveryHandlerExtraArgs: Record<string, any> = {}
|
||||
let failedTimes = 1
|
||||
let failedExact = false
|
||||
let recoveredTimes = 1
|
||||
|
||||
let script_path = ''
|
||||
let initialScriptPath = ''
|
||||
|
||||
export function openEdit(ePath: string, isFlow: boolean) {
|
||||
is_flow = isFlow
|
||||
initialPath = ePath
|
||||
itemKind = is_flow ? 'flow' : 'script'
|
||||
if (path == ePath) {
|
||||
loadSchedule()
|
||||
} else {
|
||||
path = ePath
|
||||
}
|
||||
edit = true
|
||||
drawer?.openDrawer()
|
||||
}
|
||||
|
||||
export function openNew(is_flow: boolean, initial_script_path?: string) {
|
||||
edit = false
|
||||
itemKind = is_flow ? 'flow' : 'script'
|
||||
initialScriptPath = initial_script_path ?? ''
|
||||
path = initialScriptPath
|
||||
initialPath = initialScriptPath
|
||||
script_path = initialScriptPath
|
||||
errorHandlerSelected = $enterpriseLicense ? 'slack' : 'custom'
|
||||
errorHandleritemKind = 'script'
|
||||
errorHandlerPath = undefined
|
||||
errorHandlerCustomInitialPath = undefined
|
||||
errorHandlerExtraArgs = {}
|
||||
recoveryHandlerSelected = $enterpriseLicense ? 'slack' : 'custom'
|
||||
recoveryHandlerPath = undefined
|
||||
recoveryHandlerCustomInitialPath = undefined
|
||||
recoveryHandlerItemKind = 'script'
|
||||
recoveryHandlerExtraArgs = {}
|
||||
timezone = Intl.DateTimeFormat().resolvedOptions().timeZone
|
||||
drawer?.openDrawer()
|
||||
}
|
||||
|
||||
$: is_flow = itemKind == 'flow'
|
||||
|
||||
let runnable: Script | Flow | undefined
|
||||
let args: Record<string, any> = {}
|
||||
|
||||
let isValid = true
|
||||
|
||||
let path: string = ''
|
||||
let enabled: boolean = false
|
||||
let pathError = ''
|
||||
|
||||
let validCRON = true
|
||||
$: allowSchedule = isValid && validCRON && script_path != ''
|
||||
|
||||
$: script_path != '' && loadScript(script_path)
|
||||
|
||||
// set isValid to true when a script/flow without any properties is selected
|
||||
$: runnable?.schema &&
|
||||
runnable.schema.properties &&
|
||||
Object.keys(runnable.schema.properties).length === 0 &&
|
||||
(isValid = true)
|
||||
|
||||
const dispatch = createEventDispatcher()
|
||||
|
||||
async function loadScript(p: string | undefined): Promise<void> {
|
||||
if (p) {
|
||||
if (is_flow) {
|
||||
runnable = await FlowService.getFlowByPath({ workspace: $workspaceStore!, path: p })
|
||||
} else {
|
||||
runnable = await ScriptService.getScriptByPath({ workspace: $workspaceStore!, path: p })
|
||||
}
|
||||
} else {
|
||||
runnable = undefined
|
||||
}
|
||||
}
|
||||
|
||||
let can_write = true
|
||||
async function loadSchedule(): Promise<void> {
|
||||
try {
|
||||
const s = await ScheduleService.getSchedule({
|
||||
workspace: $workspaceStore!,
|
||||
path: initialPath
|
||||
})
|
||||
enabled = s.enabled
|
||||
schedule = s.schedule
|
||||
timezone = s.timezone
|
||||
script_path = s.script_path ?? ''
|
||||
is_flow = s.is_flow
|
||||
if (s.on_failure) {
|
||||
let splitted = s.on_failure.split('/')
|
||||
errorHandleritemKind = splitted[0] as 'flow' | 'script'
|
||||
errorHandlerPath = splitted.slice(1)?.join('/')
|
||||
errorHandlerCustomInitialPath = errorHandlerPath
|
||||
failedTimes = s.on_failure_times ?? 1
|
||||
failedExact = s.on_failure_exact ?? false
|
||||
errorHandlerExtraArgs = s.on_failure_extra_args ?? {}
|
||||
if (errorHandlerPath !== slackErrorHandler) {
|
||||
errorHandlerSelected = 'custom'
|
||||
}
|
||||
} else {
|
||||
errorHandlerPath = undefined
|
||||
errorHandleritemKind = 'script'
|
||||
}
|
||||
if (s.on_recovery) {
|
||||
let splitted = s.on_recovery.split('/')
|
||||
recoveryHandlerItemKind = splitted[0] as 'flow' | 'script'
|
||||
recoveryHandlerPath = splitted.slice(1)?.join('/')
|
||||
recoveryHandlerCustomInitialPath = recoveryHandlerPath
|
||||
recoveredTimes = s.on_recovery_times ?? 1
|
||||
recoveryHandlerExtraArgs = s.on_recovery_extra_args ?? {}
|
||||
if (recoveryHandlerPath !== slackRecoveryHandler) {
|
||||
recoveryHandlerSelected = 'custom'
|
||||
}
|
||||
} else {
|
||||
recoveryHandlerPath = undefined
|
||||
recoveryHandlerItemKind = 'script'
|
||||
}
|
||||
args = s.args ?? {}
|
||||
can_write = canWrite(s.path, s.extra_perms, $userStore)
|
||||
} catch (err) {
|
||||
sendUserToast(`Could not load schedule: ${err}`, true)
|
||||
}
|
||||
}
|
||||
|
||||
async function scheduleScript(): Promise<void> {
|
||||
if (errorHandlerSelected === 'slack' && !emptyString(errorHandlerPath)) {
|
||||
errorHandlerExtraArgs['slack'] = '$res:f/slack_bot/bot_token'
|
||||
}
|
||||
if (recoveryHandlerSelected === 'slack' && !emptyString(recoveryHandlerPath)) {
|
||||
recoveryHandlerExtraArgs['slack'] = '$res:f/slack_bot/bot_token'
|
||||
}
|
||||
if (edit) {
|
||||
await ScheduleService.updateSchedule({
|
||||
workspace: $workspaceStore!,
|
||||
path: initialPath,
|
||||
requestBody: {
|
||||
schedule: formatCron(schedule),
|
||||
timezone,
|
||||
args,
|
||||
on_failure: errorHandlerPath ? `${errorHandleritemKind}/${errorHandlerPath}` : undefined,
|
||||
on_failure_times: failedTimes,
|
||||
on_failure_exact: failedExact,
|
||||
on_failure_extra_args: errorHandlerPath ? errorHandlerExtraArgs : undefined,
|
||||
on_recovery: recoveryHandlerPath
|
||||
? `${recoveryHandlerItemKind}/${recoveryHandlerPath}`
|
||||
: undefined,
|
||||
on_recovery_times: recoveredTimes,
|
||||
on_recovery_extra_args: recoveryHandlerPath ? recoveryHandlerExtraArgs : {}
|
||||
}
|
||||
})
|
||||
sendUserToast(`Schedule ${path} updated`)
|
||||
} else {
|
||||
await ScheduleService.createSchedule({
|
||||
workspace: $workspaceStore!,
|
||||
requestBody: {
|
||||
path,
|
||||
schedule: formatCron(schedule),
|
||||
timezone,
|
||||
script_path,
|
||||
is_flow,
|
||||
args,
|
||||
enabled: true,
|
||||
on_failure: errorHandlerPath ? `${errorHandleritemKind}/${errorHandlerPath}` : undefined,
|
||||
on_failure_times: failedTimes,
|
||||
on_failure_exact: failedExact,
|
||||
on_failure_extra_args: errorHandlerPath ? errorHandlerExtraArgs : undefined,
|
||||
on_recovery: recoveryHandlerPath
|
||||
? `${recoveryHandlerItemKind}/${recoveryHandlerPath}`
|
||||
: undefined,
|
||||
on_recovery_times: recoveredTimes,
|
||||
on_recovery_extra_args: recoveryHandlerPath ? recoveryHandlerExtraArgs : {}
|
||||
}
|
||||
})
|
||||
sendUserToast(`Schedule ${path} created`)
|
||||
}
|
||||
dispatch('update')
|
||||
drawer.closeDrawer()
|
||||
}
|
||||
|
||||
$: {
|
||||
if ($workspaceStore) {
|
||||
if (edit && path != '') {
|
||||
loadSchedule()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let drawer: Drawer
|
||||
</script>
|
||||
|
||||
<Drawer size="900px" bind:this={drawer}>
|
||||
<DrawerContent
|
||||
title={edit ? `Edit schedule ${initialPath}` : 'New schedule'}
|
||||
on:close={drawer.closeDrawer}
|
||||
>
|
||||
<svelte:fragment slot="actions">
|
||||
{#if edit}
|
||||
<div class="mr-8">
|
||||
<Button
|
||||
size="sm"
|
||||
variant="border"
|
||||
startIcon={{ icon: faList }}
|
||||
disabled={!allowSchedule || pathError != '' || emptyString(script_path)}
|
||||
href={`/runs/${script_path}`}
|
||||
>
|
||||
View Runs
|
||||
</Button>
|
||||
</div>
|
||||
<div class="mr-8 center-center -mt-2">
|
||||
<Toggle
|
||||
disabled={!can_write}
|
||||
checked={enabled}
|
||||
options={{ right: 'enable', left: 'disable' }}
|
||||
on:change={async (e) => {
|
||||
await ScheduleService.setScheduleEnabled({
|
||||
path: initialPath,
|
||||
workspace: $workspaceStore ?? '',
|
||||
requestBody: { enabled: e.detail }
|
||||
})
|
||||
sendUserToast(`${e.detail ? 'enabled' : 'disabled'} schedule ${initialPath}`)
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{/if}
|
||||
<Button
|
||||
startIcon={{ icon: faSave }}
|
||||
disabled={!allowSchedule ||
|
||||
pathError != '' ||
|
||||
emptyString(script_path) ||
|
||||
(errorHandlerSelected == 'slack' &&
|
||||
!emptyString(errorHandlerPath) &&
|
||||
emptyString(errorHandlerExtraArgs['channel']))}
|
||||
on:click={scheduleScript}
|
||||
>
|
||||
{edit ? 'Save' : 'Schedule'}
|
||||
</Button>
|
||||
</svelte:fragment>
|
||||
|
||||
<div class="flex flex-col gap-8">
|
||||
{#if !edit}
|
||||
<Section label="Metadata">
|
||||
<Path
|
||||
checkInitialPathExistence
|
||||
bind:error={pathError}
|
||||
bind:path
|
||||
{initialPath}
|
||||
namePlaceholder="schedule"
|
||||
kind="schedule"
|
||||
/>
|
||||
</Section>
|
||||
{/if}
|
||||
<Section label="Schedule">
|
||||
<svelte:fragment slot="header">
|
||||
<Tooltip>Schedules use CRON syntax. Seconds are mandatory.</Tooltip>
|
||||
</svelte:fragment>
|
||||
<CronInput disabled={!can_write} bind:schedule bind:timezone bind:validCRON />
|
||||
</Section>
|
||||
<Section label="Runnable">
|
||||
{#if !edit}
|
||||
<p class="text-xs mb-1 text-tertiary">
|
||||
Pick a script or flow to be triggered by the schedule<Required required={true} />
|
||||
</p>
|
||||
<ScriptPicker
|
||||
disabled={initialScriptPath != '' || !can_write}
|
||||
initialPath={initialScriptPath}
|
||||
kinds={[Script.kind.SCRIPT]}
|
||||
allowFlow={true}
|
||||
bind:itemKind
|
||||
bind:scriptPath={script_path}
|
||||
/>
|
||||
{:else}
|
||||
<Alert type="info" title="Runnable path cannot be edited">
|
||||
Once a schedule is created, the runnable path cannot be changed. However, when renaming
|
||||
a script or a flow, the runnable path will automatically update itself.
|
||||
</Alert>
|
||||
<div class="my-2" />
|
||||
<ScriptPicker
|
||||
disabled
|
||||
initialPath={script_path}
|
||||
scriptPath={script_path}
|
||||
allowFlow={true}
|
||||
{itemKind}
|
||||
/>
|
||||
{/if}
|
||||
<div class="mt-6">
|
||||
{#if runnable}
|
||||
{#if runnable?.schema && runnable.schema.properties && Object.keys(runnable.schema.properties).length > 0}
|
||||
<SchemaForm disabled={!can_write} schema={runnable.schema} bind:isValid bind:args />
|
||||
{:else}
|
||||
<div class="text-xs texg-gray-700">
|
||||
This {is_flow ? 'flow' : 'script'} takes no argument
|
||||
</div>
|
||||
{/if}
|
||||
{:else}
|
||||
<div class="text-xs texg-gray-700 my-2">
|
||||
Pick a {is_flow ? 'flow' : 'script'} and fill its argument here
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</Section>
|
||||
<Section label="Error handler">
|
||||
<ErrorOrRecoveryHandler
|
||||
isEditable={can_write}
|
||||
handlersOnlyForEe={['slack']}
|
||||
showScriptHelpText={true}
|
||||
bind:handlerSelected={errorHandlerSelected}
|
||||
bind:handlerPath={errorHandlerPath}
|
||||
customInitialScriptPath={errorHandlerCustomInitialPath}
|
||||
slackHandlerScriptPath={slackErrorHandler}
|
||||
slackToggleText="Alert channel on error"
|
||||
customScriptTemplate="/scripts/add?hub=hub%2F2420%2Fwindmill%2Fschedule_error_handler_template"
|
||||
bind:customHandlerKind={errorHandleritemKind}
|
||||
bind:handlerExtraArgs={errorHandlerExtraArgs}
|
||||
>
|
||||
<svelte:fragment slot="custom-tab-tooltip">
|
||||
<Tooltip>
|
||||
<div class="flex gap-20 items-start mt-3">
|
||||
<div class="text-sm"
|
||||
>The following args will be passed to the error handler:
|
||||
<ul class="mt-1 ml-2">
|
||||
<li><b>path</b>: The path of the script or flow that failed.</li>
|
||||
<li><b>is_flow</b>: Whether the runnable is a flow.</li>
|
||||
<li><b>schedule_path</b>: The path of the schedule.</li>
|
||||
<li><b>error</b>: The error details.</li>
|
||||
<li
|
||||
><b>failed_times</b>: Minimum number of times the schedule failed before
|
||||
calling the error handler.</li
|
||||
>
|
||||
<li><b>started_at</b>: The start datetime of the latest job that failed.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</Tooltip>
|
||||
</svelte:fragment>
|
||||
</ErrorOrRecoveryHandler>
|
||||
|
||||
<div class="flex flex-row items-center justify-between">
|
||||
<div class="flex flex-row items-center mt-4 font-semibold text-sm gap-2">
|
||||
<p class={emptyString(errorHandlerPath) ? 'text-tertiary' : ''}
|
||||
>{#if !$enterpriseLicense}<span class="text-normal text-2xs">(ee only)</span>{/if} Triggered
|
||||
when schedule failed</p
|
||||
>
|
||||
<select
|
||||
class="!w-14"
|
||||
bind:value={failedExact}
|
||||
disabled={!$enterpriseLicense || emptyString(errorHandlerPath)}
|
||||
>
|
||||
<option value={false}>>=</option>
|
||||
<option value={true}>==</option>
|
||||
</select>
|
||||
<input
|
||||
type="number"
|
||||
class="!w-14 text-center {emptyString(errorHandlerPath) ? 'text-tertiary' : ''}"
|
||||
bind:value={failedTimes}
|
||||
disabled={!$enterpriseLicense}
|
||||
min="1"
|
||||
/>
|
||||
<p class={emptyString(errorHandlerPath) ? 'text-tertiary' : ''}
|
||||
>time{failedTimes > 1 ? 's in a row' : ''}</p
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
</Section>
|
||||
<Section label="Recovery handler">
|
||||
<svelte:fragment slot="header">
|
||||
<div class="flex flex-row gap-2">
|
||||
{#if !$enterpriseLicense}<span class="text-normal text-2xs">(ee only)</span>{/if}
|
||||
</div>
|
||||
</svelte:fragment>
|
||||
|
||||
<ErrorOrRecoveryHandler
|
||||
isEditable={can_write && !emptyString($enterpriseLicense)}
|
||||
handlersOnlyForEe={[]}
|
||||
bind:handlerSelected={recoveryHandlerSelected}
|
||||
bind:handlerPath={recoveryHandlerPath}
|
||||
customInitialScriptPath={recoveryHandlerCustomInitialPath}
|
||||
slackHandlerScriptPath={slackRecoveryHandler}
|
||||
slackToggleText="Alert channel when error recovered"
|
||||
customScriptTemplate="/scripts/add?hub=hub%2F2421%2Fwindmill%2Fschedule_recovery_handler_template"
|
||||
bind:customHandlerKind={recoveryHandlerItemKind}
|
||||
bind:handlerExtraArgs={recoveryHandlerExtraArgs}
|
||||
>
|
||||
<svelte:fragment slot="custom-tab-tooltip">
|
||||
<Tooltip>
|
||||
<div class="flex gap-20 items-start mt-3">
|
||||
<div class=" text-sm"
|
||||
>The following args will be passed to the recovery handler:
|
||||
<ul class="mt-1 ml-2">
|
||||
<li><b>path</b>: The path of the script or flow that recovered.</li>
|
||||
<li><b>is_flow</b>: Whether the runnable is a flow.</li>
|
||||
<li><b>schedule_path</b>: The path of the schedule.</li>
|
||||
<li><b>error</b>: The error of the last job that errored</li>
|
||||
<li><b>error_started_at</b>: The start datetime of the last job that errored</li
|
||||
>
|
||||
<li
|
||||
><b>success_times</b>: The number of times the schedule succeeded before
|
||||
calling the recovery handler.</li
|
||||
>
|
||||
<li><b>success_result</b>: The result of the latest successful job</li>
|
||||
<li
|
||||
><b>success_started_at</b>: The start datetime of the latest successful job</li
|
||||
>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</Tooltip>
|
||||
</svelte:fragment>
|
||||
</ErrorOrRecoveryHandler>
|
||||
|
||||
<div class="flex flex-row items-center justify-between">
|
||||
<div
|
||||
class="flex flex-row items-center mt-5 font-semibold text-sm {emptyString(
|
||||
recoveryHandlerPath
|
||||
)
|
||||
? 'text-tertiary'
|
||||
: ''}"
|
||||
>
|
||||
<p>Triggered when schedule recovered</p>
|
||||
<input
|
||||
type="number"
|
||||
class="!w-14 mx-2 text-center"
|
||||
bind:value={recoveredTimes}
|
||||
min="1"
|
||||
/>
|
||||
<p>time{recoveredTimes > 1 ? 's in a row' : ''}</p>
|
||||
</div>
|
||||
</div>
|
||||
</Section>
|
||||
</div>
|
||||
</DrawerContent>
|
||||
</Drawer>
|
||||
@@ -157,6 +157,7 @@
|
||||
job.mem_peak = previewJobUpdates.mem_peak
|
||||
}
|
||||
if ((previewJobUpdates.running ?? false) || (previewJobUpdates.completed ?? false)) {
|
||||
console.log({ a: previewJobUpdates.running, b: previewJobUpdates.completed })
|
||||
job = await JobService.getJob({ workspace: workspace!, id })
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
'other',
|
||||
'bun'
|
||||
]
|
||||
const nativeTags = ['nativets', 'postgresql', 'mysql', 'graphql', 'snowflake']
|
||||
const nativeTags = ['nativets', 'postgresql', 'mysql', 'graphql', 'snowflake', 'bigquery']
|
||||
|
||||
let newTag: string = ''
|
||||
$: selected = nconfig?.dedicated_worker != undefined ? 'dedicated' : 'normal'
|
||||
|
||||
@@ -9,9 +9,18 @@ export async function getResourceTypes() {
|
||||
if (rts) {
|
||||
return rts
|
||||
} else {
|
||||
const nrts = await ResourceService.listResourceTypeNames({ workspace: get(workspaceStore)! })
|
||||
resourceTypesStore.set(nrts)
|
||||
return nrts
|
||||
let workspace = get(workspaceStore)
|
||||
if (workspace){
|
||||
try {
|
||||
const nrts = await ResourceService.listResourceTypeNames({ workspace: workspace })
|
||||
resourceTypesStore.set(nrts)
|
||||
return nrts
|
||||
} catch (e) {
|
||||
return ["error_fetching_names"]
|
||||
}
|
||||
} else {
|
||||
return ['workspace_is_undefined']
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import { derived, type Readable, writable } from 'svelte/store'
|
||||
import type { UserWorkspaceList } from '$lib/gen/models/UserWorkspaceList.js'
|
||||
import type { TokenResponse } from './gen'
|
||||
import type { IntrospectionQuery } from 'graphql'
|
||||
import { resourceTypesStore } from './components/resourceTypesStore'
|
||||
|
||||
export interface UserExt {
|
||||
email: string
|
||||
@@ -108,6 +109,7 @@ export const dbSchemas = writable<DBSchemas>({})
|
||||
export function switchWorkspace(workspace: string | undefined) {
|
||||
localStorage.removeItem('flow')
|
||||
localStorage.removeItem('app')
|
||||
resourceTypesStore.set(undefined)
|
||||
workspaceStore.set(workspace)
|
||||
}
|
||||
|
||||
@@ -115,6 +117,7 @@ export function clearStores(): void {
|
||||
localStorage.removeItem('flow')
|
||||
localStorage.removeItem('app')
|
||||
localStorage.removeItem('workspace')
|
||||
resourceTypesStore.set(undefined)
|
||||
userStore.set(undefined)
|
||||
workspaceStore.set(undefined)
|
||||
usersWorkspaceStore.set(undefined)
|
||||
|
||||
@@ -10,6 +10,9 @@
|
||||
import Tooltip from '$lib/components/Tooltip.svelte'
|
||||
import FlowGraph from '$lib/components/graph/FlowGraph.svelte'
|
||||
import SchemaForm from '$lib/components/SchemaForm.svelte'
|
||||
import { workspaceStore } from '$lib/stores'
|
||||
|
||||
$workspaceStore = $page.params.workspace
|
||||
|
||||
let job: Job | undefined = undefined
|
||||
let currentApprovers: { resume_id: number; approver: string }[] = []
|
||||
@@ -145,7 +148,7 @@
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
<h2 class="mt-4">Flow arguments</h2>
|
||||
<h2 class="mt-4 mb-2">Flow arguments</h2>
|
||||
|
||||
<JobArgs args={job?.args} />
|
||||
<div class="mt-8">
|
||||
|
||||
Reference in New Issue
Block a user