diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e5f1a48803..0ee5367875 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -49,7 +49,7 @@ jobs: with: ref: benchmarks - name: benchmark - timeout-minutes: 10 + timeout-minutes: 20 run: deno run --unstable -A -r https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json diff --git a/backend/.sqlx/query-0715955b7e98cc669a88eca6556cd46b7f9c07fcf32a24f85b69720b54e6e95f.json b/backend/.sqlx/query-0715955b7e98cc669a88eca6556cd46b7f9c07fcf32a24f85b69720b54e6e95f.json deleted file mode 100644 index 08e88b9c96..0000000000 --- a/backend/.sqlx/query-0715955b7e98cc669a88eca6556cd46b7f9c07fcf32a24f85b69720b54e6e95f.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT mem_peak FROM queue WHERE id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "mem_peak", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - true - ] - }, - "hash": "0715955b7e98cc669a88eca6556cd46b7f9c07fcf32a24f85b69720b54e6e95f" -} diff --git a/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json b/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json index 7718e05ccf..1b8084742c 100644 --- a/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json +++ b/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json @@ -37,6 +37,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-1f040850c2a82bc09789226b167c43fd4935cfbb4951760a4d527665b70a5ac7.json b/backend/.sqlx/query-1f040850c2a82bc09789226b167c43fd4935cfbb4951760a4d527665b70a5ac7.json deleted file mode 100644 index 74f5a78f7b..0000000000 --- a/backend/.sqlx/query-1f040850c2a82bc09789226b167c43fd4935cfbb4951760a4d527665b70a5ac7.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT result, id\n FROM completed_job\n WHERE id = ANY($1)\n AND workspace_id = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "result", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "UuidArray", - "Text" - ] - }, - "nullable": [ - true, - false - ] - }, - "hash": "1f040850c2a82bc09789226b167c43fd4935cfbb4951760a4d527665b70a5ac7" -} diff --git a/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json b/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json index 9d082a6772..bf591ef11c 100644 --- a/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json +++ b/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json @@ -67,6 +67,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json b/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json index 1166260449..a5dee163e5 100644 --- a/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json +++ b/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json @@ -28,6 +28,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-541ebd3bac65431237cf3b882dfdcd61ca97c253d9754d05bba59fda89841067.json b/backend/.sqlx/query-541ebd3bac65431237cf3b882dfdcd61ca97c253d9754d05bba59fda89841067.json deleted file mode 100644 index f03861669b..0000000000 --- a/backend/.sqlx/query-541ebd3bac65431237cf3b882dfdcd61ca97c253d9754d05bba59fda89841067.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT value, approver, resume_id FROM resume_job WHERE job = $1 ORDER BY created_at ASC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "value", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "approver", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "resume_id", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - false, - true, - false - ] - }, - "hash": "541ebd3bac65431237cf3b882dfdcd61ca97c253d9754d05bba59fda89841067" -} diff --git a/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json b/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json index eabf671894..1517c8d1d4 100644 --- a/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json +++ b/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json @@ -42,6 +42,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json b/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json index c52efca4c0..bfe7c41f64 100644 --- a/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json +++ b/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json @@ -42,6 +42,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json b/backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json index 73e6e17c76..dabac76bfe 100644 --- a/backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json +++ b/backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json @@ -46,6 +46,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-913c634de21d46b8841f8a7c25c408da7c572f9e685db6351848cbf6e9253efc.json b/backend/.sqlx/query-8b221f0d08f3304364e56f5c4894fea42975b03d21904655d724a35163413d29.json similarity index 59% rename from backend/.sqlx/query-913c634de21d46b8841f8a7c25c408da7c572f9e685db6351848cbf6e9253efc.json rename to backend/.sqlx/query-8b221f0d08f3304364e56f5c4894fea42975b03d21904655d724a35163413d29.json index e767997903..97e023a089 100644 --- a/backend/.sqlx/query-913c634de21d46b8841f8a7c25c408da7c572f9e685db6351848cbf6e9253efc.json +++ b/backend/.sqlx/query-8b221f0d08f3304364e56f5c4894fea42975b03d21904655d724a35163413d29.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE queue SET mem_peak = GREATEST($1, mem_peak), last_ping = now() WHERE id = $2 RETURNING canceled", + "query": "UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled", "describe": { "columns": [ { @@ -19,5 +19,5 @@ false ] }, - "hash": "913c634de21d46b8841f8a7c25c408da7c572f9e685db6351848cbf6e9253efc" + "hash": "8b221f0d08f3304364e56f5c4894fea42975b03d21904655d724a35163413d29" } diff --git a/backend/.sqlx/query-8cea673a5b17bc7cf671b539c2c1eb34731c77aaf42dc12e45472eeb134887d0.json b/backend/.sqlx/query-8cea673a5b17bc7cf671b539c2c1eb34731c77aaf42dc12e45472eeb134887d0.json deleted file mode 100644 index 37fcf0f7ac..0000000000 --- a/backend/.sqlx/query-8cea673a5b17bc7cf671b539c2c1eb34731c77aaf42dc12e45472eeb134887d0.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT raw_flow->'modules'->$1::int->'stop_after_if'->>'expr' as stop_early_expr,\n (raw_flow->'modules'->$1::int->'stop_after_if'->>'skip_if_stopped')::bool as skip_if_stopped,\n args \n FROM queue\n WHERE id = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "stop_early_expr", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "skip_if_stopped", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "args", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid" - ] - }, - "nullable": [ - null, - null, - true - ] - }, - "hash": "8cea673a5b17bc7cf671b539c2c1eb34731c77aaf42dc12e45472eeb134887d0" -} diff --git a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json index 61da7601af..8b139967a6 100644 --- a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json +++ b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json @@ -40,6 +40,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json b/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json index fead4ba250..6308bf3bb2 100644 --- a/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json +++ b/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json @@ -60,6 +60,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-a227548b6604c56bfc15eb780bd8ee72a89dc6701a50f5048e928bd87baa7b9a.json b/backend/.sqlx/query-a227548b6604c56bfc15eb780bd8ee72a89dc6701a50f5048e928bd87baa7b9a.json deleted file mode 100644 index 204c4adca6..0000000000 --- a/backend/.sqlx/query-a227548b6604c56bfc15eb780bd8ee72a89dc6701a50f5048e928bd87baa7b9a.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT result FROM completed_job WHERE id = ANY($1) AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "result", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "UuidArray", - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "a227548b6604c56bfc15eb780bd8ee72a89dc6701a50f5048e928bd87baa7b9a" -} diff --git a/backend/.sqlx/query-ad9ccde8d831461f1f312d530867714577a825cd40d6655ed61b0b343b5d4482.json b/backend/.sqlx/query-ad9ccde8d831461f1f312d530867714577a825cd40d6655ed61b0b343b5d4482.json new file mode 100644 index 0000000000..0a8ed659a4 --- /dev/null +++ b/backend/.sqlx/query-ad9ccde8d831461f1f312d530867714577a825cd40d6655ed61b0b343b5d4482.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource\n (workspace_id, path, value, resource_type)\n VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "ad9ccde8d831461f1f312d530867714577a825cd40d6655ed61b0b343b5d4482" +} diff --git a/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json b/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json index c90719118a..38f81da395 100644 --- a/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json +++ b/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json @@ -42,6 +42,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-c2849e67b9fea0dc46e6d7000f5a0c9dab89ae80a183d9255f8fdb356b4bc61c.json b/backend/.sqlx/query-c2849e67b9fea0dc46e6d7000f5a0c9dab89ae80a183d9255f8fdb356b4bc61c.json deleted file mode 100644 index 3c43b2689e..0000000000 --- a/backend/.sqlx/query-c2849e67b9fea0dc46e6d7000f5a0c9dab89ae80a183d9255f8fdb356b4bc61c.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT result FROM completed_job WHERE id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "result", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - true - ] - }, - "hash": "c2849e67b9fea0dc46e6d7000f5a0c9dab89ae80a183d9255f8fdb356b4bc61c" -} diff --git a/backend/.sqlx/query-b69f747eae8b13a0a8d6914a3a7ad322554fcfe62cd28c5f0a475f18dd770d61.json b/backend/.sqlx/query-dda02bee4e15e0f3a2b0cb17ca86ae7ee3a4c0413d3484d3bc960796b39cd650.json similarity index 57% rename from backend/.sqlx/query-b69f747eae8b13a0a8d6914a3a7ad322554fcfe62cd28c5f0a475f18dd770d61.json rename to backend/.sqlx/query-dda02bee4e15e0f3a2b0cb17ca86ae7ee3a4c0413d3484d3bc960796b39cd650.json index 8078d5496f..b9554a787e 100644 --- a/backend/.sqlx/query-b69f747eae8b13a0a8d6914a3a7ad322554fcfe62cd28c5f0a475f18dd770d61.json +++ b/backend/.sqlx/query-dda02bee4e15e0f3a2b0cb17ca86ae7ee3a4c0413d3484d3bc960796b39cd650.json @@ -1,15 +1,20 @@ { "db_name": "PostgreSQL", - "query": "SELECT substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3", + "query": "SELECT running, substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3", "describe": { "columns": [ { "ordinal": 0, + "name": "running", + "type_info": "Bool" + }, + { + "ordinal": 1, "name": "logs", "type_info": "Text" }, { - "ordinal": 1, + "ordinal": 2, "name": "mem_peak", "type_info": "Int4" } @@ -22,9 +27,10 @@ ] }, "nullable": [ + false, null, true ] }, - "hash": "b69f747eae8b13a0a8d6914a3a7ad322554fcfe62cd28c5f0a475f18dd770d61" + "hash": "dda02bee4e15e0f3a2b0cb17ca86ae7ee3a4c0413d3484d3bc960796b39cd650" } diff --git a/backend/.sqlx/query-f8b34e09453d51d3df5be20652938a890ee353fec64ee73c312a3352da7f7515.json b/backend/.sqlx/query-f8b34e09453d51d3df5be20652938a890ee353fec64ee73c312a3352da7f7515.json deleted file mode 100644 index 13526abb60..0000000000 --- a/backend/.sqlx/query-f8b34e09453d51d3df5be20652938a890ee353fec64ee73c312a3352da7f7515.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT error_handler FROM workspace_settings WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "error_handler", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "f8b34e09453d51d3df5be20652938a890ee353fec64ee73c312a3352da7f7515" -} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index ee4984d026..7b2c380900 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -7383,6 +7383,7 @@ version = "1.183.0" dependencies = [ "anyhow", "async-recursion", + "axum", "bigdecimal 0.4.1", "chrono", "chrono-tz", @@ -7441,6 +7442,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "regex", + "reqwest", "rsmq_async", "rust_decimal", "serde", @@ -7452,7 +7454,6 @@ dependencies = [ "tracing", "urlencoding", "uuid 1.4.1", - "windmill-api-client", "windmill-audit", "windmill-common", "windmill-parser", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 899dc2aa74..c182439f72 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -11,7 +11,6 @@ members = [ "./windmill-worker", "./windmill-common", "./windmill-audit", - "./windmill-api-client", "./parsers/windmill-parser", "./parsers/windmill-parser-ts", "./parsers/windmill-parser-wasm", @@ -45,7 +44,6 @@ tokio.workspace = true dotenv.workspace = true windmill-common = { workspace = true, features = ["tracing_init"] } windmill-api.workspace = true -windmill-api-client.workspace = true windmill-worker.workspace = true futures.workspace = true tracing.workspace = true @@ -72,10 +70,11 @@ reqwest.workspace = true windmill-queue.workspace = true axum.workspace = true serde.workspace = true +windmill-api-client.workspace = true + [workspace.dependencies] windmill-api = { path = "./windmill-api" } -windmill-api-client = { path = "./windmill-api-client" } windmill-queue = { path = "./windmill-queue" } windmill-worker = { path = "./windmill-worker" } windmill-common = { path = "./windmill-common" } @@ -88,6 +87,7 @@ windmill-parser-go = { path = "./parsers/windmill-parser-go" } windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } windmill-parser-sql = { path = "./parsers/windmill-parser-sql" } windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" } +windmill-api-client = { path = "./windmill-api-client" } axum = { version = "^0", features = ["headers"] } headers = "^0" diff --git a/backend/README.md b/backend/README.md index 216784dd53..a184290c5d 100644 --- a/backend/README.md +++ b/backend/README.md @@ -5,15 +5,14 @@ contains files used to build the "root" binary. ## Components -| name | description | -| --------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| [windmill-api](./windmill-api/) | The API server, exposing functionality to other components and the frontend | -| [windmill-api-client](./windmill-api-client/) | An autogenerated Rust API client, used by other components to talk to the API | -| [windmill-audit](./windmill-audit/) | Contains audit functionality, allowing different components to record important actions | -| [windmill-common](./windmill-common/) | Common code shared by all crates | -| [windmill-queue](./windmill-queue/) | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers | -| [windmill-worker](./windmill-worker/) | The worker. Used to process and execute flows & jobs. | -| [parsers](./parsers/) | Contains code to parse signatures in different langauges. | +| name | description | +| ------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| [windmill-api](./windmill-api/) | The API server, exposing functionality to other components and the frontend | +| [windmill-audit](./windmill-audit/) | Contains audit functionality, allowing different components to record important actions | +| [windmill-common](./windmill-common/) | Common code shared by all crates | +| [windmill-queue](./windmill-queue/) | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers | +| [windmill-worker](./windmill-worker/) | The worker. Used to process and execute flows & jobs. | +| [parsers](./parsers/) | Contains code to parse signatures in different langauges. | ### Compile sqlx for offline ci diff --git a/backend/migrations/20231016084043_addback_bigquery.down.sql b/backend/migrations/20231016084043_addback_bigquery.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20231016084043_addback_bigquery.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20231016084043_addback_bigquery.up.sql b/backend/migrations/20231016084043_addback_bigquery.up.sql new file mode 100644 index 0000000000..77fd1784ea --- /dev/null +++ b/backend/migrations/20231016084043_addback_bigquery.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +UPDATE config set config = '{"worker_tags": ["nativets", "postgresq", "mysql", "graphql", "snowflake", "bigquery"]}'::jsonb where name = 'worker__native' and config = '{"worker_tags": ["nativets", "postgresql", "mysql", "graphql" +, "snowflake"]}'::jsonb ; \ No newline at end of file diff --git a/backend/res.json b/backend/res.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 26006f9cde..31a3b12ebd 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1,6 +1,5 @@ use std::{collections::HashMap, fmt::Display, ops::Mul, str::FromStr, sync::Arc, time::Duration}; -use once_cell::sync::OnceCell; use serde::de::DeserializeOwned; use sqlx::{Pool, Postgres}; use tokio::{ @@ -639,7 +638,6 @@ async fn handle_zombie_jobs base_internal_url: base_internal_url.to_string(), token, workspace: job.workspace_id.to_string(), - client: OnceCell::new(), }; let last_ping = job.last_ping.clone(); @@ -647,6 +645,7 @@ async fn handle_zombie_jobs db, &client, &job, + 0, error::Error::ExecutionErr(format!( "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})", last_ping diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index b41ca8245b..e42d85bd3f 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1,4 +1,6 @@ use std::sync::Arc; +use windmill_api_client::types::{NewScript, NewScriptLanguage}; +use std::str::FromStr; #[cfg(feature = "enterprise")] use chrono::Timelike; @@ -7,6 +9,7 @@ use futures::StreamExt; use futures::{stream, Stream}; use serde::Deserialize; use serde_json::json; +use sqlx::types::Json; use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres}; use tokio::sync::RwLock; @@ -22,7 +25,6 @@ use sqlx::query; #[cfg(feature = "enterprise")] use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs}; -use windmill_api_client::types::{NewScript, NewScriptLanguage}; use windmill_common::worker::WORKER_CONFIG; use windmill_common::{ @@ -34,7 +36,6 @@ use windmill_common::{ use windmill_queue::PushIsolationLevel; use serde::Serialize; -use std::str::FromStr; #[derive(Debug, sqlx::FromRow, Serialize)] pub struct CompletedJob { @@ -872,12 +873,12 @@ impl RunJob { async fn push(self, db: &Pool) -> Uuid { let RunJob { payload, args } = self; let tx = PushIsolationLevel::IsolatedRoot(db.clone(), None); - let (uuid, tx) = windmill_queue::push::( + let (uuid, tx) = windmill_queue::push::<_, rsmq_async::MultiplexedRsmq>( &db, tx, "test-workspace", payload, - args, + Json(args), /* user */ "test-user", /* email */ "test@windmill.dev", /* permissioned_as */ "u/test-user".to_string(), @@ -1874,7 +1875,7 @@ async fn test_invalid_first_step(db: Pool) { assert_eq!( job.json_result().unwrap(), - serde_json::json!( {"error": {"name": "InternalErr", "message": "Expected an array value, found: {}"}}) + serde_json::json!( {"error": {"name": "InternalErr", "message": "Expected an array value, found: invalid type: map, expected a sequence at line 1 column 0"}}) ); } diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index f48e27462a..32c8fd127f 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -24,7 +24,7 @@ use axum::{ use hyper::StatusCode; use magic_crypt::MagicCryptTrait; use serde::{Deserialize, Serialize}; -use serde_json::{json, Map, Value}; +use serde_json::{json, value::RawValue}; use sha2::{Digest, Sha256}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::{types::Uuid, FromRow}; @@ -40,7 +40,7 @@ use windmill_common::{ http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath, }, }; -use windmill_queue::{push, PushIsolationLevel, QueueTransaction}; +use windmill_queue::{push, PushArgs, PushIsolationLevel, QueueTransaction}; pub fn workspaced_service() -> Router { Router::new() @@ -124,7 +124,7 @@ pub struct AppWithLastVersionAndDraft { pub draft_only: Option, } -pub type StaticFields = Map; +pub type StaticFields = HashMap>; #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] #[serde(rename_all = "lowercase")] @@ -518,7 +518,7 @@ async fn create_app( tx, &w_id, JobPayload::AppDependencies { path: app.path.clone(), version: v_id }, - serde_json::Map::new(), + PushArgs::empty(), &authed.username, &authed.email, windmill_common::users::username_to_permissioned_as(&authed.username), @@ -761,7 +761,7 @@ async fn update_app( tx, &w_id, JobPayload::AppDependencies { path: npath.clone(), version: v_id }, - serde_json::Map::new(), + PushArgs::empty(), &authed.username, &authed.email, windmill_common::users::username_to_permissioned_as(&authed.username), @@ -798,7 +798,7 @@ async fn update_app( #[derive(Debug, Deserialize, Clone)] pub struct ExecuteApp { - pub args: Map, + pub args: Box, // - script: script/ // - flow: flow/ pub path: Option, @@ -900,17 +900,17 @@ async fn execute_component( } }; - let (job_payload, args, tag) = match &payload { + let (job_payload, args, tag) = match payload { ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => { let content = &raw_code.content; let payload = JobPayload::Code(raw_code.clone()); let path = digest(content); - let args = build_args(policy, component, path, args)?; + let args = build_args(policy, &component, path, args)?; (payload, args, None) } ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => { - let (payload, tag) = get_payload_tag_from_prefixed_path(path, &db, &w_id).await?; - let args = build_args(policy, component, path.to_string(), args)?; + let (payload, tag) = get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?; + let args = build_args(policy, &component, path.to_string(), args)?; (payload, args, tag) } _ => unreachable!(), @@ -1002,11 +1002,17 @@ fn build_args( policy: Policy, component: &str, path: String, - args: &Map, -) -> Result> { + args: Box, +) -> Result>> { // disallow var and res access in args coming from the user for security reasons - args.into_iter() - .try_for_each(|x| disallow_var_res_access(x.1))?; + { + let args_str = args.to_string(); + if args_str.contains("$var:") || args_str.contains("$res:") { + return Err(Error::BadRequest(format!( + "For security reasons, variable or resource access is not allowed as dynamic argument" + ))); + } + } let key = format!("{}:{}", component, &path); let static_args = policy .triggerables @@ -1015,7 +1021,7 @@ fn build_args( .map(|x| x.clone()) .or_else(|| { if matches!(policy.execution_mode, ExecutionMode::Viewer) { - Some(Map::new()) + Some(HashMap::new()) } else { None } @@ -1023,26 +1029,9 @@ fn build_args( .ok_or_else(|| { Error::BadRequest(format!("path {} is not allowed in the app policy", path)) })?; - let mut args = args.clone(); + let mut extra = HashMap::new(); for (k, v) in static_args { - args.insert(k.to_string(), v.to_owned()); - } - Ok(args) -} - -fn disallow_var_res_access(args: &serde_json::Value) -> Result<()> { - match args { - Value::Object(v) => v.into_iter().try_for_each(|x| disallow_var_res_access(x.1)), - Value::Array(arr) => arr.into_iter().try_for_each(|v| disallow_var_res_access(v)), - Value::String(s) => { - if s.starts_with("$var:") || s.starts_with("$res:") { - Err(Error::BadRequest(format!( - "For security reasons, variable or resource access is not allowed as dynamic argument" - ))) - } else { - Ok(()) - } - } - _ => Ok(()), + extra.insert(k.to_string(), v.to_owned()); } + Ok(PushArgs { extra, args: sqlx::types::Json(args) }) } diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index f25d07b4b6..1223f93ede 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -6,23 +6,24 @@ * LICENSE-AGPL for a copy of the license. */ +use std::collections::HashMap; + use axum::{ - extract::{Extension, Path, Query}, + extract::{Extension, Path}, routing::{get, post, put}, Router, }; -use hyper::{HeaderMap, StatusCode}; -use serde::Deserialize; +use hyper::StatusCode; +use serde_json::value::RawValue; +use sqlx::types::Json; use windmill_common::{ db::UserDB, error::{JsonResult, Result}, utils::{not_found_if_none, StripPath}, }; +use windmill_queue::PushArgs; -use crate::{ - db::{ApiAuthed, DB}, - jobs::{add_include_headers, add_raw_string, JsonOrForm}, -}; +use crate::db::{ApiAuthed, DB}; const KEEP_LAST: i64 = 8; @@ -85,21 +86,12 @@ pub async fn new_payload( Ok(StatusCode::CREATED) } -#[derive(Deserialize, Clone)] -pub struct IncludeHeaderQuery { - include_header: Option, -} - pub async fn update_payload( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, - Query(run_query): Query, - headers: HeaderMap, - JsonOrForm(args, raw_string): JsonOrForm, + args: PushArgs>>, ) -> Result { let mut tx = db.begin().await?; - let args = add_include_headers(&run_query.include_header, headers, args.unwrap_or_default()); - let args = add_raw_string(raw_string, args); sqlx::query!( " @@ -110,7 +102,7 @@ pub async fn update_payload( ", &w_id, &path.to_path(), - serde_json::json!(args), + Json(args) as Json>>>, ) .execute(&mut *tx) .await?; diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 5c55f9f891..2fc65b811a 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -38,6 +38,7 @@ use windmill_common::{ scripts::Schema, utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, StripPath}, }; +use windmill_queue::PushArgs; use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction}; pub fn workspaced_service() -> Router { @@ -290,7 +291,7 @@ async fn create_flow( tx, &w_id, JobPayload::FlowDependencies { path: nf.path.clone() }, - serde_json::Map::new(), + PushArgs::empty(), &authed.username, &authed.email, windmill_common::users::username_to_permissioned_as(&authed.username), @@ -482,7 +483,7 @@ async fn update_flow( tx, &w_id, JobPayload::FlowDependencies { path: nf.path.clone() }, - serde_json::Map::new(), + PushArgs::empty(), &authed.username, &authed.email, windmill_common::users::username_to_permissioned_as(&authed.username), diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 0a7426f6f9..5dd961022a 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -6,6 +6,8 @@ * LICENSE-AGPL for a copy of the license. */ +use std::collections::HashMap; + use crate::db::ApiAuthed; use crate::{ @@ -19,13 +21,12 @@ use axum::{ extract::{FromRequest, Json, Path, Query}, response::{IntoResponse, Response}, routing::{get, post}, - Extension, Form, RequestExt, Router, + Extension, Router, }; use base64::Engine; -use bytes::Bytes; use chrono::Utc; use hmac::Mac; -use hyper::{header::CONTENT_TYPE, http, HeaderMap, Request, StatusCode}; +use hyper::{http, Request, StatusCode}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sql_builder::{prelude::*, quote, SqlBuilder}; use sqlx::types::JsonRawValue; @@ -46,7 +47,7 @@ use windmill_common::{ users::username_to_permissioned_as, utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath}, }; -use windmill_queue::{job_is_complete, push, PushIsolationLevel}; +use windmill_queue::{empty_args, job_is_complete, push, PushArgs, PushIsolationLevel}; pub fn workspaced_service() -> Router { let cors = CorsLayer::new() @@ -87,18 +88,6 @@ pub fn workspaced_service() -> Router { .head(|| async { "" }) .layer(cors.clone()), ) - .route( - "/openai_sync/p/*script_path", - post(openai_sync_script_by_path) - .head(|| async { "" }) - .layer(cors.clone()), - ) - .route( - "/openai_sync/f/*script_path", - post(openai_sync_flow_by_path) - .head(|| async { "" }) - .layer(cors.clone()), - ) .route( "/run/h/:hash", post(run_job_by_hash) @@ -190,11 +179,16 @@ pub fn global_root_service() -> Router { Router::new().route("/db_clock", get(get_db_clock)) } +#[derive(Deserialize)] +struct JsonPath { + pub json_path: Option, +} async fn get_result_by_id( Extension(db): Extension, Path((w_id, flow_id, node_id)): Path<(String, Uuid, String)>, -) -> windmill_common::error::JsonResult { - let res = windmill_queue::get_result_by_id(db, w_id, flow_id, node_id).await?; + Query(JsonPath { json_path }): Query, +) -> windmill_common::error::JsonResult> { + let res = windmill_queue::get_result_by_id(db, w_id, flow_id, node_id, json_path).await?; Ok(Json(res)) } @@ -350,7 +344,7 @@ async fn get_job( ) -> error::Result { let cjob_option = sqlx::query("SELECT id, workspace_id, parent_job, created_by, created_at, duration_ms, success, script_hash, script_path, - args, CASE WHEN pg_column_size(result) < 2000000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind, env_id, + CASE WHEN pg_column_size(args) < 2000000 THEN args ELSE '{\"reason\": \"WINDMILL_TOO_BIG\"}'::jsonb END as args, CASE WHEN pg_column_size(result) < 2000000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind, env_id, schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language, started_at, is_skipped, raw_lock, email, visible_to_owner, mem_peak, tag FROM completed_job WHERE id = $1 AND workspace_id = $2") @@ -363,7 +357,11 @@ async fn get_job( Ok(Json(job).into_response()) } else { let job_o = sqlx::query_as::<_, QueuedJob>( - "SELECT * + "SELECT id, workspace_id, parent_job, created_by, created_at, started_at, scheduled_for, running, + script_hash, script_path, CASE WHEN pg_column_size(args) < 2000000 THEN args ELSE '{\"reason\": \"WINDMILL_TOO_BIG\"}'::jsonb END as args, logs, raw_code, canceled, canceled_by, canceled_reason, last_ping, + job_kind, env_id, schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language, + suspend, suspend_until, same_worker, raw_lock, pre_run_error, email, visible_to_owner, mem_peak, + root_job, leaf_jobs, tag, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, cache_ttl FROM queue WHERE id = $1 AND workspace_id = $2", ) .bind(id) @@ -407,7 +405,7 @@ pub struct CompletedJob<'rows> { pub script_hash: Option, #[serde(skip_serializing_if = "Option::is_none")] pub script_path: Option, - pub args: Option, + pub args: Option<&'rows JsonRawValue>, #[serde(skip_serializing_if = "Option::is_none", borrow)] pub result: Option<&'rows JsonRawValue>, #[serde(skip_serializing_if = "Option::is_none")] @@ -502,21 +500,12 @@ pub struct RunJobQuery { scheduled_for: Option>, scheduled_in_secs: Option, parent_job: Option, - include_header: Option, invisible_to_owner: Option, queue_limit: Option, payload: Option, job_id: Option, } -lazy_static::lazy_static! { - static ref INCLUDE_HEADERS: Vec = std::env::var("INCLUDE_HEADERS") - .ok().map(|x| x - .split(',') - .map(|s| s.to_string()) - .collect()).unwrap_or_default(); -} - impl RunJobQuery { async fn get_scheduled_for<'c>( &self, @@ -531,38 +520,6 @@ impl RunJobQuery { Ok(None) } } - - fn add_include_headers( - &self, - headers: HeaderMap, - args: serde_json::Map, - ) -> serde_json::Map { - return add_include_headers(&self.include_header, headers, args); - } -} - -pub fn add_include_headers( - include_header: &Option, - headers: HeaderMap, - mut args: serde_json::Map, -) -> serde_json::Map { - let whitelist = include_header - .as_ref() - .map(|s| s.split(",").map(|s| s.to_string()).collect::>()) - .unwrap_or_default(); - - whitelist - .iter() - .chain(INCLUDE_HEADERS.iter()) - .for_each(|h| { - if let Some(v) = headers.get(h) { - args.insert( - h.to_string().to_lowercase().replace('-', "_"), - serde_json::Value::String(v.to_str().unwrap().to_string()), - ); - } - }); - args } #[derive(Deserialize)] @@ -1485,7 +1442,7 @@ struct Preview { content: Option, kind: Option, path: Option, - args: Option>, + args: Option>, language: Option, tag: Option, } @@ -1494,78 +1451,10 @@ struct Preview { struct PreviewFlow { value: FlowValue, path: Option, - args: Option>, + args: Option>, tag: Option, } -pub struct JsonOrForm( - pub Option>, - pub Option, -); - -#[axum::async_trait] -impl FromRequest for JsonOrForm -where - S: Send + Sync, -{ - type Rejection = Response; - - async fn from_request( - req: Request, - _state: &S, - ) -> Result { - let content_type_header = req.headers().get(CONTENT_TYPE); - let content_type = content_type_header.and_then(|value| value.to_str().ok()); - if content_type.is_none() || content_type.unwrap().starts_with("application/json") { - if req - .uri() - .query() - .map(|x| x.contains("raw=true")) - .unwrap_or(false) - { - let bytes = Bytes::from_request(req, _state) - .await - .map_err(IntoResponse::into_response)?; - let str = String::from_utf8(bytes.to_vec()).map_err(|e| { - Error::BadRequest(format!("invalid utf8: {}", e)).into_response() - })?; - let payload = - serde_json::from_str::>(&str).map_err(|e| { - Error::BadRequest(format!("invalid json: {}", e)).into_response() - })?; - return match payload { - Some(serde_json::Value::Object(map)) => Ok(Self(Some(map), Some(str))), - None => Ok(Self(None, Some(str))), - Some(x) => { - let mut map = serde_json::Map::new(); - map.insert("body".to_string(), x); - Ok(Self(Some(map), Some(str))) - } - }; - } else { - let Json(payload): Json> = - req.extract().await.map_err(IntoResponse::into_response)?; - return match payload { - Some(serde_json::Value::Object(map)) => Ok(Self(Some(map), None)), - None => Ok(Self(None, None)), - Some(x) => { - let mut map = serde_json::Map::new(); - map.insert("body".to_string(), x); - Ok(Self(Some(map), None)) - } - }; - } - } else if content_type - .unwrap() - .starts_with("application/x-www-form-urlencoded") - { - let Form(payload) = req.extract().await.map_err(IntoResponse::into_response)?; - return Ok(Self(Some(payload), None)); - } else { - Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response()) - } - } -} pub struct QueryOrBody(pub Option); #[axum::async_trait] @@ -1673,8 +1562,7 @@ pub async fn run_flow_by_path( Extension(rsmq): Extension>, Path((w_id, flow_path)): Path<(String, StripPath)>, Query(run_query): Query, - headers: HeaderMap, - JsonOrForm(args, raw_string): JsonOrForm, + args: PushArgs>>, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -1691,8 +1579,6 @@ pub async fn run_flow_by_path( .flatten(); check_tag_available_for_workspace(&w_id, &tag).await?; let scheduled_for = run_query.get_scheduled_for(&db).await?; - let args = run_query.add_include_headers(headers, args.unwrap_or_default()); - let args = add_raw_string(raw_string, args); let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq); let (uuid, tx) = push( &db, @@ -1728,8 +1614,7 @@ pub async fn run_job_by_path( Extension(rsmq): Extension>, Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, - headers: HeaderMap, - JsonOrForm(args, raw_string): JsonOrForm, + args: PushArgs>>, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -1740,8 +1625,6 @@ pub async fn run_job_by_path( let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?; let scheduled_for = run_query.get_scheduled_for(&db).await?; - let args = run_query.add_include_headers(headers, args.unwrap_or_default()); - let args = add_raw_string(raw_string, args); check_tag_available_for_workspace(&w_id, &tag).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq); @@ -1934,12 +1817,14 @@ pub async fn run_wait_result_job_by_path_get( .map(decode_payload) .map(|x| x.map_err(|e| Error::InternalErr(e.to_string()))); - let args = if let Some(payload) = payload_r { + let payload_args = if let Some(payload) = payload_r { payload? } else { - serde_json::Map::new() + HashMap::new() }; + let args = PushArgs { extra: payload_args, args: sqlx::types::Json(empty_args()) }; + check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; let script_path = script_path.to_path(); check_scopes(&authed, || format!("run:script/{script_path}"))?; @@ -1983,7 +1868,7 @@ pub async fn run_wait_result_flow_by_path_get( Extension(user_db): Extension, Extension(db): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, - headers: HeaderMap, + Query(run_query): Query, ) -> error::JsonResult { #[cfg(feature = "enterprise")] @@ -1998,22 +1883,16 @@ pub async fn run_wait_result_flow_by_path_get( .map(decode_payload) .map(|x| x.map_err(|e| Error::InternalErr(e.to_string()))); - let args = if let Some(payload) = payload_r { + let payload_args = if let Some(payload) = payload_r { payload? } else { - serde_json::Map::new() + HashMap::new() }; + + let args = PushArgs { extra: payload_args, args: sqlx::types::Json(HashMap::new()) }; + run_wait_result_flow_by_path_internal( - db, - run_query, - flow_path, - authed, - rsmq, - user_db, - headers, - Some(args), - None, - w_id, + db, run_query, flow_path, authed, rsmq, user_db, args, w_id, ) .await } @@ -2025,8 +1904,7 @@ pub async fn run_wait_result_script_by_path( Extension(db): Extension, Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, - headers: HeaderMap, - JsonOrForm(args, raw_string): JsonOrForm, + args: PushArgs>>, ) -> error::JsonResult { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -2039,58 +1917,7 @@ pub async fn run_wait_result_script_by_path( rsmq, user_db, w_id, - headers, args, - raw_string, - ) - .await -} - -fn convert_from_openai_json( - json: Option>, -) -> error::Result>> { - if let Some(m) = json { - let mut new_json = serde_json::Map::new(); - let input_keys = m - .get("inputKeys") - .and_then(|x| x.as_array()) - .map(|x| x.to_owned()) - .unwrap_or_default(); - let input_values = m - .get("inputValues") - .and_then(|x| x.as_array()) - .map(|x| x.to_owned()) - .unwrap_or_default(); - for (k, v) in input_keys.into_iter().zip(input_values.into_iter()) { - new_json.insert(k.as_str().unwrap_or_else(|| "invalid_key").to_string(), v); - } - Ok(Some(new_json)) - } else { - Ok(None) - } -} - -pub async fn openai_sync_script_by_path( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(rsmq): Extension>, - Extension(db): Extension, - Path((w_id, script_path)): Path<(String, StripPath)>, - Query(run_query): Query, - headers: HeaderMap, - JsonOrForm(args, raw_string): JsonOrForm, -) -> error::JsonResult { - run_wait_result_script_by_path_internal( - db, - run_query, - script_path, - authed, - rsmq, - user_db, - w_id, - headers, - convert_from_openai_json(args)?, - raw_string, ) .await } @@ -2103,9 +1930,7 @@ async fn run_wait_result_script_by_path_internal( rsmq: Option, user_db: UserDB, w_id: String, - headers: HeaderMap, - args: Option>, - raw_string: Option, + args: PushArgs>>, ) -> Result, Error> { check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; let script_path = script_path.to_path(); @@ -2113,8 +1938,6 @@ async fn run_wait_result_script_by_path_internal( let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?; - let args = run_query.add_include_headers(headers, args.unwrap_or_default()); - let args = add_raw_string(raw_string, args); check_tag_available_for_workspace(&w_id, &tag).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq); @@ -2153,8 +1976,7 @@ pub async fn run_wait_result_script_by_hash( Extension(db): Extension, Path((w_id, script_hash)): Path<(String, ScriptHash)>, Query(run_query): Query, - headers: HeaderMap, - JsonOrForm(args, raw_string): JsonOrForm, + args: PushArgs>>, ) -> error::JsonResult { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -2173,8 +1995,6 @@ pub async fn run_wait_result_script_by_hash( ) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?; check_scopes(&authed, || format!("run:script/{path}"))?; - let args = run_query.add_include_headers(headers, args.unwrap_or_default()); - let args = add_raw_string(raw_string, args); check_tag_available_for_workspace(&w_id, &tag).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq); @@ -2214,31 +2034,6 @@ pub async fn run_wait_result_script_by_hash( run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_hash))).await } -pub async fn openai_sync_flow_by_path( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(rsmq): Extension>, - Extension(db): Extension, - Path((w_id, flow_path)): Path<(String, StripPath)>, - Query(run_query): Query, - headers: HeaderMap, - JsonOrForm(args, raw_string): JsonOrForm, -) -> error::JsonResult { - run_wait_result_flow_by_path_internal( - db, - run_query, - flow_path, - authed, - rsmq, - user_db, - headers, - convert_from_openai_json(args)?, - raw_string, - w_id, - ) - .await -} - pub async fn run_wait_result_flow_by_path( authed: ApiAuthed, Extension(user_db): Extension, @@ -2246,14 +2041,13 @@ pub async fn run_wait_result_flow_by_path( Extension(db): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, Query(run_query): Query, - headers: HeaderMap, - JsonOrForm(args, raw_string): JsonOrForm, + args: PushArgs>>, ) -> error::JsonResult { #[cfg(feature = "enterprise")] check_license_key_valid().await?; run_wait_result_flow_by_path_internal( - db, run_query, flow_path, authed, rsmq, user_db, headers, args, raw_string, w_id, + db, run_query, flow_path, authed, rsmq, user_db, args, w_id, ) .await } @@ -2265,9 +2059,7 @@ async fn run_wait_result_flow_by_path_internal( authed: ApiAuthed, rsmq: Option, user_db: UserDB, - headers: HeaderMap, - args: Option>, - raw_string: Option, + args: PushArgs>>, w_id: String, ) -> Result, Error> { check_queue_too_long(&db, run_query.queue_limit).await?; @@ -2276,8 +2068,7 @@ async fn run_wait_result_flow_by_path_internal( check_scopes(&authed, || format!("run:flow/{flow_path}"))?; let scheduled_for = run_query.get_scheduled_for(&db).await?; - let args = run_query.add_include_headers(headers, args.unwrap_or_default()); - let args = add_raw_string(raw_string, args); + let tag = sqlx::query_scalar!( "SELECT tag from flow WHERE path = $1 and workspace_id = $2", flow_path, @@ -2324,7 +2115,6 @@ async fn run_preview_job( Extension(rsmq): Extension>, Path(w_id): Path, Query(run_query): Query, - headers: HeaderMap, Json(preview): Json, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] @@ -2337,7 +2127,6 @@ async fn run_preview_job( )); } let scheduled_for = run_query.get_scheduled_for(&db).await?; - let args = run_query.add_include_headers(headers, preview.args.unwrap_or_default()); check_tag_available_for_workspace(&w_id, &preview.tag).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq); @@ -2358,7 +2147,7 @@ async fn run_preview_job( cache_ttl: None, }), }, - args, + preview.args.unwrap_or_default(), &authed.username, &authed.email, username_to_permissioned_as(&authed.username), @@ -2436,7 +2225,7 @@ async fn add_batch_jobs( value: batch_info.flow_value.clone().unwrap(), path: None, }, - serde_json::Map::new(), + PushArgs::empty(), &authed.username, &authed.email, username_to_permissioned_as(&authed.username), @@ -2519,7 +2308,6 @@ async fn run_preview_flow_job( Extension(rsmq): Extension>, Path(w_id): Path, Query(run_query): Query, - headers: HeaderMap, Json(raw_flow): Json, ) -> error::Result<(StatusCode, String)> { check_scopes(&authed, || format!("runflow"))?; @@ -2529,7 +2317,6 @@ async fn run_preview_flow_job( )); } let scheduled_for = run_query.get_scheduled_for(&db).await?; - let args = run_query.add_include_headers(headers, raw_flow.args.unwrap_or_default()); check_tag_available_for_workspace(&w_id, &raw_flow.tag).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq); @@ -2538,7 +2325,7 @@ async fn run_preview_flow_job( tx, &w_id, JobPayload::RawFlow { value: raw_flow.value, path: raw_flow.path }, - args, + raw_flow.args.unwrap_or_default(), &authed.username, &authed.email, username_to_permissioned_as(&authed.username), @@ -2569,8 +2356,7 @@ pub async fn run_job_by_hash( Extension(rsmq): Extension>, Path((w_id, script_hash)): Path<(String, ScriptHash)>, Query(run_query): Query, - headers: HeaderMap, - JsonOrForm(args, raw_string): JsonOrForm, + args: PushArgs>>, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -2588,8 +2374,7 @@ pub async fn run_job_by_hash( check_scopes(&authed, || format!("run:script/{path}"))?; let scheduled_for = run_query.get_scheduled_for(&db).await?; - let args = run_query.add_include_headers(headers, args.unwrap_or_default()); - let args = add_raw_string(raw_string, args); + check_tag_available_for_workspace(&w_id, &tag).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq); @@ -2648,21 +2433,22 @@ async fn get_job_update( Path((w_id, id)): Path<(String, Uuid)>, Query(JobUpdateQuery { running, log_offset }): Query, ) -> error::JsonResult { - let mut tx = db.begin().await?; - let record = sqlx::query!( - "SELECT substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3", + "SELECT running, substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3", log_offset, &w_id, &id ) - .fetch_optional(&mut *tx) + .fetch_optional(&db) .await?; if let Some(record) = record { - tx.commit().await?; Ok(Json(JobUpdate { - running: if !running { Some(true) } else { None }, + running: if !running && record.running { + Some(true) + } else { + None + }, completed: None, new_logs: record.logs, mem_peak: record.mem_peak, @@ -2675,10 +2461,9 @@ async fn get_job_update( &w_id, &id ) - .fetch_optional(&mut *tx) + .fetch_optional(&db) .await?; let logs = not_found_if_none(logs, "Job Update", id.to_string())?; - tx.commit().await?; Ok(Json(JobUpdate { running: Some(false), completed: Some(true), @@ -2851,7 +2636,10 @@ async fn get_completed_job<'a>( Extension(db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::Result { - let job_o = sqlx::query("SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2") + let job_o = sqlx::query("SELECT id, workspace_id, parent_job, created_by, created_at, duration_ms, success, script_hash, script_path, + CASE WHEN pg_column_size(args) < 2000000 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args, CASE WHEN pg_column_size(result) < 2000000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind, env_id, + schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language, started_at, is_skipped, + raw_lock, email, visible_to_owner, mem_peak, tag FROM completed_job WHERE id = $1 AND workspace_id = $2") .bind(id) .bind(w_id) .fetch_optional(&db) @@ -2875,13 +2663,29 @@ impl<'a> IntoResponse for RawResult<'a> { async fn get_completed_job_result( Extension(db): Extension, Path((w_id, id)): Path<(String, Uuid)>, + Query(JsonPath { json_path }): Query, ) -> error::Result { - let result_o = + let result_o = if let Some(json_path) = json_path { + sqlx::query( + "SELECT result #> $3 as result FROM completed_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(id) + .bind(w_id) + .bind( + json_path + .split(".") + .map(|x| x.to_string()) + .collect::>(), + ) + .fetch_optional(&db) + .await? + } else { sqlx::query("SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2") .bind(id) .bind(w_id) .fetch_optional(&db) - .await?; + .await? + }; let result = not_found_if_none(result_o, "Completed Job", id.to_string())?; Ok(RawResult::from_row(&result)?.into_response()) diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 7bc1b39818..42d5317077 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -201,7 +201,6 @@ pub async fn run_server( users::global_service().layer(Extension(argon2.clone())), ) .nest("/settings", settings::global_service()) - .nest("/jobs", jobs::global_root_service()) .nest("/workers", workers::global_service()) .nest("/configs", configs::global_service()) .nest("/scripts", scripts::global_service()) @@ -212,6 +211,7 @@ pub async fn run_server( .nest("/schedules", schedule::global_service()) .route_layer(from_extractor::()) .route_layer(from_extractor::()) + .nest("/jobs", jobs::global_root_service()) .nest( "/saml", saml::global_service().layer(Extension(Arc::new(sp_extension.0))), diff --git a/backend/windmill-api/src/oauth2.rs b/backend/windmill-api/src/oauth2.rs index 1709eda27c..2ba62ea3de 100644 --- a/backend/windmill-api/src/oauth2.rs +++ b/backend/windmill-api/src/oauth2.rs @@ -903,7 +903,7 @@ async fn slack_command( tx, &settings.workspace_id, payload, - map, + sqlx::types::Json(map), &form.user_name, &settings.slack_email, "g/slack".to_string(), diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 5999823468..00e14a78d2 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -21,7 +21,7 @@ use axum::{ }; use hyper::StatusCode; use serde::{Deserialize, Serialize}; -use serde_json::Value; +use serde_json::{value::RawValue, Value}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::{FromRow, Postgres, Transaction}; use uuid::Uuid; @@ -117,7 +117,7 @@ pub struct ListableResource { #[derive(Deserialize)] pub struct CreateResource { pub path: String, - pub value: Option, + pub value: Option>, pub description: Option, pub resource_type: String, } @@ -125,7 +125,7 @@ pub struct CreateResource { struct EditResource { path: Option, description: Option, - value: Option, + value: Option>, } #[derive(Deserialize)] @@ -496,6 +496,9 @@ async fn create_resource( check_path_conflict(&mut tx, &w_id, &resource.path).await?; } + let res_value = resource.value.unwrap_or_default(); + let raw_json = sqlx::types::Json(res_value.as_ref()); + sqlx::query!( "INSERT INTO resource (workspace_id, path, value, description, resource_type) @@ -503,7 +506,7 @@ async fn create_resource( DO UPDATE SET value = $3, description = $4, resource_type = $5", w_id, resource.path, - resource.value, + raw_json as sqlx::types::Json<&RawValue>, resource.description, resource.resource_type, ) diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index f35174d53e..d7519602e6 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -46,7 +46,9 @@ use windmill_common::{ not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath, }, }; -use windmill_queue::{self, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction}; +use windmill_queue::{ + self, schedule::push_scheduled_job, PushArgs, PushIsolationLevel, QueueTransaction, +}; const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20; @@ -592,7 +594,7 @@ async fn create_script( tx, &w_id, JobPayload::Dependencies { hash, dependencies, language: ns.language, path: ns.path }, - serde_json::Map::new(), + PushArgs::empty(), &authed.username, &authed.email, username_to_permissioned_as(&authed.username), diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 1ee00f6ea2..cafb81f535 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -549,7 +549,7 @@ async fn run_slack_message_test_job( false, w_id.as_str(), &format!("script/{}", req.hub_script_path.as_str()), - &json!(fake_result), + sqlx::types::Json(&fake_result), 0, Utc::now(), Some(json!(extra_args)), diff --git a/backend/windmill-common/src/flow_status.rs b/backend/windmill-common/src/flow_status.rs index 12f54c38ea..9c4b5c6b72 100644 --- a/backend/windmill-common/src/flow_status.rs +++ b/backend/windmill-common/src/flow_status.rs @@ -11,8 +11,8 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; use uuid::Uuid; +use crate::flows::FlowValue; use crate::more_serde::default_false; -use crate::{flows::FlowValue, more_serde::is_default}; const MINUTES: Duration = Duration::from_secs(60); const HOURS: Duration = MINUTES.saturating_mul(60); @@ -20,21 +20,24 @@ const HOURS: Duration = MINUTES.saturating_mul(60); pub const MAX_RETRY_ATTEMPTS: u16 = 1000; pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6); +pub fn is_retry_default(v: &RetryStatus) -> bool { + v.fail_count == 0 && v.failed_jobs.is_empty() +} + #[derive(Serialize, Deserialize, Debug)] pub struct FlowStatus { pub step: i32, pub modules: Vec, pub failure_module: FlowStatusModuleWParent, #[serde(default)] - #[serde(skip_serializing_if = "is_default")] + #[serde(skip_serializing_if = "is_retry_default")] pub retry: RetryStatus, } -#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct RetryStatus { pub fail_count: u16, - pub previous_result: Option, pub failed_jobs: Vec, } @@ -47,7 +50,6 @@ pub struct Iterator { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct BranchAllStatus { pub branch: usize, - pub previous_result: serde_json::Value, pub len: usize, } @@ -189,7 +191,7 @@ impl FlowStatus { .unwrap_or_else(|| "failure".to_string()), }, }, - retry: RetryStatus { fail_count: 0, previous_result: None, failed_jobs: vec![] }, + retry: RetryStatus { fail_count: 0, failed_jobs: vec![] }, } } diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 87b246eb4b..d54b1a2a8a 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -12,6 +12,7 @@ use std::{ }; use serde::{self, Deserialize, Serialize, Serializer}; +use serde_json::value::RawValue; use crate::{ more_serde::{ @@ -171,7 +172,7 @@ pub struct Suspend { pub struct Mock { pub enabled: bool, #[serde(skip_serializing_if = "Option::is_none")] - pub return_value: Option, + pub return_value: Option>, } #[derive(Deserialize, Serialize, Debug, Clone)] @@ -203,7 +204,7 @@ impl FlowModule { } } -#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[derive(Deserialize, Serialize, Debug, Clone)] #[serde( tag = "type", rename_all(serialize = "lowercase", deserialize = "lowercase") diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 4074cde6ae..74b071374f 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -1,5 +1,8 @@ +use std::collections::HashMap; + use serde::{Deserialize, Serialize}; -use sqlx::{Pool, Postgres, Transaction}; +use serde_json::value::RawValue; +use sqlx::{types::Json, Pool, Postgres, Transaction}; use uuid::Uuid; use crate::{ @@ -27,7 +30,7 @@ pub enum JobKind { Noop, } -#[derive(Debug, sqlx::FromRow, Serialize, Clone)] +#[derive(sqlx::FromRow, Debug, Serialize, Clone)] pub struct QueuedJob { pub workspace_id: String, pub id: Uuid, @@ -43,7 +46,7 @@ pub struct QueuedJob { pub script_hash: Option, #[serde(skip_serializing_if = "Option::is_none")] pub script_path: Option, - pub args: Option, + pub args: Option>>>, #[serde(skip_serializing_if = "Option::is_none")] pub logs: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -95,6 +98,14 @@ pub struct QueuedJob { } impl QueuedJob { + pub fn get_args(&self) -> HashMap> { + if let Some(args) = self.args.as_ref() { + args.0.clone() + } else { + HashMap::new() + } + } + pub fn script_path(&self) -> &str { self.script_path .as_ref() @@ -112,9 +123,7 @@ impl QueuedJob { self.script_path() ) } -} -impl QueuedJob { pub fn parse_raw_flow(&self) -> Option { self.raw_flow .as_ref() diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 80ce2e3cac..96fe028b50 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -3,6 +3,7 @@ use std::{collections::HashMap, sync::Arc}; use itertools::Itertools; use regex::Regex; use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; use tokio::sync::RwLock; use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::ServerConfig, DB}; @@ -260,3 +261,13 @@ pub struct WorkerConfig { pub additional_python_paths: Option>, pub pip_local_dependencies: Option>, } + +pub fn to_raw_value(result: &T) -> Box { + serde_json::value::to_raw_value(result) + .unwrap_or_else(|_| RawValue::from_string("{}".to_string()).unwrap()) +} + +pub fn to_raw_value_owned(result: serde_json::Value) -> Box { + serde_json::value::to_raw_value(&result) + .unwrap_or_else(|_| RawValue::from_string("{}".to_string()).unwrap()) +} diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index 12b3ff2140..4d68857d7a 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -37,4 +37,5 @@ tokio.workspace = true futures-core.workspace = true itertools.workspace = true async-recursion.workspace = true -bigdecimal.workspace = true \ No newline at end of file +bigdecimal.workspace = true +axum.workspace = true \ No newline at end of file diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 0609c8fc1c..f90a0097e0 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -10,12 +10,23 @@ use std::{collections::HashMap, vec}; use anyhow::Context; use async_recursion::async_recursion; +use axum::{ + body::Bytes, + extract::FromRequest, + http::Request, + response::{IntoResponse, Response}, + Form, RequestExt, +}; use bigdecimal::ToPrimitive; use chrono::{DateTime, Duration, Utc}; -use reqwest::Client; +use reqwest::{ + header::{HeaderMap, CONTENT_TYPE}, + Client, StatusCode, +}; use rsmq_async::RsmqConnection; -use serde_json::json; -use sqlx::{Pool, Postgres, Transaction}; +use serde::Serialize; +use serde_json::{json, value::RawValue}; +use sqlx::{types::Json, FromRow, Pool, Postgres, Transaction}; #[cfg(feature = "benchmark")] use std::time::Instant; use tracing::{instrument, Instrument}; @@ -36,7 +47,7 @@ use windmill_common::{ schedule::{schedule_to_user, Schedule}, scripts::{ScriptHash, ScriptLang}, users::{username_to_permissioned_as, SUPERADMIN_SECRET_EMAIL}, - worker::WORKER_CONFIG, + worker::{to_raw_value, WORKER_CONFIG}, DB, METRICS_ENABLED, }; @@ -120,6 +131,7 @@ pub async fn cancel_job<'c: 'async_recursion>( &db, &job_running, format!("canceled by {username}: (force cancel: {force_cancel})"), + job_running.mem_peak.unwrap_or(0), &e, None, rsmq.clone(), @@ -166,20 +178,39 @@ pub async fn cancel_job<'c: 'async_recursion>( Ok((tx, Some(id))) } +#[derive(Serialize)] +pub struct WrappedError { + pub error: T, +} + #[instrument(level = "trace", skip_all)] -pub async fn add_completed_job_error( +pub async fn add_completed_job_error< + T: Serialize + Send + Sync, + R: rsmq_async::RsmqConnection + Clone + Send, +>( db: &Pool, queued_job: &QueuedJob, logs: String, - e: &serde_json::Value, + mem_peak: i32, + e: T, metrics: Option, rsmq: Option, -) -> Result { +) -> Result, Error> { if *METRICS_ENABLED { metrics.map(|m| m.worker_execution_failed.inc()); } - let result = serde_json::json!({ "error": e }); - let _ = add_completed_job(db, &queued_job, false, false, &result, logs, rsmq).await?; + let result = WrappedError { error: e }; + let _ = add_completed_job( + db, + &queued_job, + false, + false, + Json(&result), + logs, + mem_peak, + rsmq, + ) + .await?; Ok(result) } @@ -206,13 +237,17 @@ lazy_static::lazy_static! { } #[instrument(level = "trace", skip_all, name = "add_completed_job")] -pub async fn add_completed_job( +pub async fn add_completed_job< + T: Serialize + Send + Sync, + R: rsmq_async::RsmqConnection + Clone + Send, +>( db: &Pool, queued_job: &QueuedJob, success: bool, skipped: bool, - result: &serde_json::Value, + result: Json<&T>, logs: String, + mem_peak: i32, rsmq: Option, ) -> Result { // tracing::error!("Start"); @@ -245,14 +280,11 @@ pub async fn add_completed_job( None }; - let mem_peak = sqlx::query_scalar!("SELECT mem_peak FROM queue WHERE id = $1", &queued_job.id) - .fetch_optional(db) - .await - .ok() - .flatten() - .flatten(); let mut tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into(); - let job_id = queued_job.id.clone(); + let job_id = queued_job.id; + // tracing::error!("1 {:?}", start.elapsed()); + + let mem_peak = mem_peak.max(queued_job.mem_peak.unwrap_or(0)); let _duration: i64 = sqlx::query_scalar!( "INSERT INTO completed_job AS cj ( workspace_id @@ -298,8 +330,8 @@ pub async fn add_completed_job( success, queued_job.script_hash.map(|x| x.0), queued_job.script_path, - queued_job.args, - result, + &queued_job.args as &Option>>>, + result as Json<&T>, logs, queued_job.raw_code, queued_job.raw_lock, @@ -317,15 +349,18 @@ pub async fn add_completed_job( duration as Option, queued_job.email, queued_job.visible_to_owner, - mem_peak, + if mem_peak > 0 { Some(mem_peak) } else { None }, queued_job.tag, ) .fetch_one(&mut tx) .await .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e}")))?; + // tracing::error!("2 {:?}", start.elapsed()); // tracing::error!("Added completed job {:#?}", queued_job); tx = delete_job(tx, &queued_job.workspace_id, job_id).await?; + // tracing::error!("3 {:?}", start.elapsed()); + if !queued_job.is_flow_step && queued_job.schedule_path.is_some() && queued_job.script_path.is_some() @@ -337,7 +372,7 @@ pub async fn add_completed_job( queued_job.script_path.as_ref().unwrap(), &queued_job.workspace_id, success, - &result, + result, job_id, queued_job.started_at.unwrap_or(chrono::Utc::now()), ) @@ -399,7 +434,7 @@ pub async fn add_completed_job( && queued_job.parent_job.is_none() && !success { - if let Err(e) = send_error_to_global_handler(rsmq.clone(), &queued_job, db, &result).await { + if let Err(e) = send_error_to_global_handler(rsmq.clone(), &queued_job, db, result).await { tracing::error!( "Could not run global error handler for job {}: {}", &queued_job.id, @@ -407,8 +442,7 @@ pub async fn add_completed_job( ); } - if let Err(e) = - send_error_to_workspace_handler(rsmq.clone(), &queued_job, db, &result).await + if let Err(e) = send_error_to_workspace_handler(rsmq.clone(), &queued_job, db, result).await { tracing::error!( "Could not run workspace error handler for job {}: {}", @@ -419,16 +453,20 @@ pub async fn add_completed_job( } tracing::debug!("Added completed job {}", queued_job.id); - // tracing::error!("{:?}", start.elapsed()); + // tracing::error!("4 {:?}", start.elapsed()); Ok(queued_job.id) } -pub async fn run_error_handler( +pub async fn run_error_handler< + 'a, + T: Serialize + Send + Sync, + R: rsmq_async::RsmqConnection + Clone + Send, +>( rsmq: Option, queued_job: &QueuedJob, db: &Pool, - result: &serde_json::Value, + result: Json<&'a T>, error_handler_path: &str, error_handler_extra_args: Option, is_global: bool, @@ -437,16 +475,22 @@ pub async fn run_error_handler( let script_w_id = if is_global { "admins" } else { w_id }; // script workspace id let job_id = queued_job.id; let (job_payload, tag) = script_path_to_payload(&error_handler_path, db, script_w_id).await?; - let mut args = result.as_object().unwrap().clone(); - args.insert("workspace_id".to_string(), json!(w_id)); - args.insert("job_id".to_string(), json!(job_id)); - args.insert("path".to_string(), json!(queued_job.script_path)); - args.insert("is_flow".to_string(), json!(queued_job.raw_flow.is_some())); - args.insert("email".to_string(), json!(queued_job.email)); + + let mut extra = HashMap::new(); + extra.insert("workspace_id".to_string(), to_raw_value(&w_id)); + extra.insert("job_id".to_string(), to_raw_value(&job_id)); + extra.insert("path".to_string(), to_raw_value(&queued_job.script_path)); + extra.insert( + "is_flow".to_string(), + to_raw_value(&queued_job.raw_flow.is_some()), + ); + extra.insert("email".to_string(), to_raw_value(&queued_job.email)); if let Some(extra_args) = error_handler_extra_args { if let serde_json::Value::Object(args_m) = extra_args { - args.extend(args_m); + for (k, v) in args_m { + extra.insert(k, to_raw_value(&v)); + } } else { return Err(error::Error::ExecutionErr( "args of scripts needs to be dict".to_string(), @@ -461,7 +505,7 @@ pub async fn run_error_handler( tx, script_w_id, job_payload, - args, + PushArgs { extra, args: result.to_owned() }, if is_global { "global" } else { "error_handler" }, if is_global { SUPERADMIN_SECRET_EMAIL @@ -497,11 +541,15 @@ pub async fn run_error_handler( Ok(()) } -pub async fn send_error_to_global_handler( +pub async fn send_error_to_global_handler< + 'a, + T: Serialize + Send + Sync, + R: rsmq_async::RsmqConnection + Clone + Send, +>( rsmq: Option, queued_job: &QueuedJob, db: &Pool, - result: &serde_json::Value, + result: Json<&'a T>, ) -> Result<(), Error> { if let Some(ref global_error_handler) = *GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE { run_error_handler( @@ -519,11 +567,16 @@ pub async fn send_error_to_global_handler( +pub async fn send_error_to_workspace_handler< + 'a, + 'c, + T: Serialize + Send + Sync, + R: rsmq_async::RsmqConnection + Clone + Send, +>( rsmq: Option, queued_job: &QueuedJob, db: &Pool, - result: &serde_json::Value, + result: Json<&'a T>, ) -> Result<(), Error> { let w_id = &queued_job.workspace_id; let mut tx = db.begin().await?; @@ -620,20 +673,25 @@ pub async fn handle_maybe_scheduled_job<'c, R: rsmq_async::RsmqConnection + Clon } } -#[derive(Clone)] +#[derive(Clone, Serialize)] struct CompletedJobSubset { success: bool, result: Option, started_at: chrono::DateTime, } -async fn apply_schedule_handlers<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>( +async fn apply_schedule_handlers< + 'a, + 'c, + T: Serialize + Send + Sync, + R: rsmq_async::RsmqConnection + Clone + Send + 'c, +>( mut tx: QueueTransaction<'c, R>, db: &Pool, schedule_path: &str, script_path: &str, w_id: &str, success: bool, - result: &serde_json::Value, + result: Json<&'a T>, job_id: Uuid, started_at: DateTime, ) -> windmill_common::error::Result> { @@ -790,7 +848,12 @@ async fn apply_schedule_handlers<'c, R: rsmq_async::RsmqConnection + Clone + Sen Ok(tx) } -pub async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>( +pub async fn handle_on_failure< + 'a, + 'c, + T: Serialize + Send + Sync, + R: rsmq_async::RsmqConnection + Clone + Send + 'c, +>( db: &Pool, tx: QueueTransaction<'c, R>, schedule_path: &str, @@ -798,7 +861,7 @@ pub async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send is_flow: bool, w_id: &str, on_failure_path: &str, - result: &serde_json::Value, + result: Json<&'a T>, failed_times: i32, started_at: DateTime, extra_args: Option, @@ -808,16 +871,18 @@ pub async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send ) -> windmill_common::error::Result<(Uuid, QueueTransaction<'c, R>)> { let (payload, tag) = get_payload_tag_from_prefixed_path(on_failure_path, db, w_id).await?; - let mut args = result.clone().as_object().unwrap().clone(); - args.insert("schedule_path".to_string(), json!(schedule_path)); - args.insert("path".to_string(), json!(script_path)); - args.insert("is_flow".to_string(), json!(is_flow)); - args.insert("started_at".to_string(), json!(started_at)); - args.insert("failed_times".to_string(), json!(failed_times)); + let mut extra = HashMap::new(); + extra.insert("schedule_path".to_string(), to_raw_value(&schedule_path)); + extra.insert("path".to_string(), to_raw_value(&script_path)); + extra.insert("is_flow".to_string(), to_raw_value(&is_flow)); + extra.insert("started_at".to_string(), to_raw_value(&started_at)); + extra.insert("failed_times".to_string(), to_raw_value(&failed_times)); if let Some(args_v) = extra_args { if let serde_json::Value::Object(args_m) = args_v { - args.extend(args_m); + for (k, v) in args_m { + extra.insert(k, to_raw_value(&v)); + } } else { return Err(error::Error::ExecutionErr( "args of scripts needs to be dict".to_string(), @@ -831,7 +896,7 @@ pub async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send tx, w_id, payload, - args, + PushArgs { extra, args: result.to_owned() }, username, email, permissioned_as, @@ -857,7 +922,20 @@ pub async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send return Ok((uuid, tx)); } -async fn handle_on_recovery<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>( +// #[derive(Serialize)] +// pub struct RecoveryValue { +// error_started_at: chrono::DateTime, +// schedule_path: String, +// path: String, +// is_flow: boolean, +// extra_args: serde_json::Value +// } +async fn handle_on_recovery< + 'a, + 'c, + T: Serialize + Send + Sync, + R: rsmq_async::RsmqConnection + Clone + Send + 'c, +>( db: &Pool, tx: QueueTransaction<'c, R>, schedule_path: &str, @@ -866,7 +944,7 @@ async fn handle_on_recovery<'c, R: rsmq_async::RsmqConnection + Clone + Send + ' w_id: &str, on_recovery_path: &str, error_job: CompletedJobSubset, - successful_job_result: &serde_json::Value, + successful_job_result: Json<&'a T>, successful_times: i32, successful_job_started_at: DateTime, extra_args: Option, @@ -886,7 +964,11 @@ async fn handle_on_recovery<'c, R: rsmq_async::RsmqConnection + Clone + Send + ' args.insert("schedule_path".to_string(), json!(schedule_path)); args.insert("path".to_string(), json!(script_path)); args.insert("is_flow".to_string(), json!(is_flow)); - args.insert("success_result".to_string(), successful_job_result.clone()); + args.insert( + "success_result".to_string(), + serde_json::from_str(&serde_json::to_string(&successful_job_result).unwrap()) + .unwrap_or_else(|_| json!("{}")), + ); args.insert("success_times".to_string(), json!(successful_times)); args.insert( "success_started_at".to_string(), @@ -1074,14 +1156,14 @@ pub async fn pull( // if using redis, only one message at a time can be poped from the queue. Process only this message and move to the next elligible job // In this case, the job might be a job from the same script path, but we can't optimise this further // if using posgtres, then we're able to re-queue the entire batch of scheduled job for this script_path, so we do it - let _requeued_job = sqlx::query_as::<_, QueuedJob>(&format!( + let requeued_job_tag = sqlx::query_scalar::<_, String>(&format!( "UPDATE queue SET running = false , started_at = null , scheduled_for = '{estimated_next_schedule_timestamp}' , logs = CASE WHEN logs IS NULL OR logs = '' THEN '{job_log_event}'::text WHEN logs LIKE '%{job_log_event}' THEN logs ELSE concat(logs, '{job_log_line_break}{job_log_event}'::text) END WHERE id = '{job_uuid}' - RETURNING *" + RETURNING tag" )) .fetch_one(&mut tx) .await @@ -1091,20 +1173,19 @@ pub async fn pull( rsmq.send_message( job_uuid.to_bytes_le().to_vec(), Option::Some(estimated_next_schedule_timestamp), - _requeued_job.tag, + requeued_job_tag, ); } tx.commit().await?; } else { // if using posgtres, then we're able to re-queue the entire batch of scheduled job for this script_path, so we do it - let _requeued_jobs = sqlx::query_as::<_, QueuedJob>(&format!( + sqlx::query(&format!( "UPDATE queue SET running = false , started_at = null , scheduled_for = '{estimated_next_schedule_timestamp}' , logs = CASE WHEN logs IS NULL OR logs = '' THEN '{job_log_event}'::text WHEN logs LIKE '%{job_log_event}' THEN logs ELSE concat(logs, '{job_log_line_break}{job_log_event}'::text) END - WHERE (id = '{job_uuid}') OR (script_path = '{job_script_path}' AND running = false AND scheduled_for <= now()) - RETURNING *" + WHERE (id = '{job_uuid}') OR (script_path = '{job_script_path}' AND running = false AND scheduled_for <= now())" )) .fetch_all(&mut tx) .await @@ -1154,7 +1235,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit< .map_err(|_| anyhow::anyhow!("Failed to parsed Redis message"))?, ); - let m2 = sqlx::query_as::<_, QueuedJob>( + let m2r = sqlx::query( "UPDATE queue SET running = true , started_at = coalesce(started_at, now()) @@ -1166,6 +1247,11 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit< .bind(uuid) .fetch_optional(db) .await?; + let m2 = if let Some(row) = m2r { + Some(QueuedJob::from_row(&row)?) + } else { + None + }; rsmq.delete_message(&tag.unwrap(), &msg.id) .await @@ -1190,7 +1276,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit< let tags = config.worker_tags.clone(); drop(config); let r = if suspend_first { - sqlx::query_as::<_, QueuedJob>("UPDATE queue + sqlx::query("UPDATE queue SET running = true , started_at = coalesce(started_at, now()) , last_ping = now() @@ -1210,14 +1296,18 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit< } else { None }; - + let r = if let Some(row) = r { + Some(QueuedJob::from_row(&row)?) + } else { + None + }; if r.is_none() { // #[cfg(feature = "benchmark")] // let instant = Instant::now(); let tags = WORKER_CONFIG.read().await.worker_tags.clone(); - let r = sqlx::query_as::<_, QueuedJob>( + let r = sqlx::query( "UPDATE queue SET running = true , started_at = coalesce(started_at, now()) @@ -1239,7 +1329,11 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit< // #[cfg(feature = "benchmark")] // println!("pull query: {:?}", instant.elapsed()); - r + if let Some(row) = r { + Some(QueuedJob::from_row(&row)?) + } else { + None + } } else { r } @@ -1247,13 +1341,19 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit< Ok(job) } +#[derive(FromRow)] +struct ResultR { + result: Option>>, +} + #[async_recursion] pub async fn get_result_by_id( db: Pool, w_id: String, flow_id: Uuid, node_id: String, -) -> error::Result { + json_path: Option, +) -> error::Result> { let flow_job_result = sqlx::query!( "SELECT leaf_jobs->$1::text as leaf_jobs, parent_job FROM queue WHERE COALESCE((SELECT root_job FROM queue WHERE id = $2), $2) = id AND workspace_id = $3", node_id, @@ -1265,7 +1365,7 @@ pub async fn get_result_by_id( let flow_job_result = windmill_common::utils::not_found_if_none( flow_job_result, - "Flow result by id", + "Flow result by id in leaf jobs", format!("{}, {}", flow_id, node_id), )?; @@ -1281,7 +1381,7 @@ pub async fn get_result_by_id( .await? .flatten() .unwrap_or(parent_job); - return get_result_by_id(db, w_id, root_job, node_id).await; + return get_result_by_id(db, w_id, root_job, node_id, json_path).await; } let result_id = windmill_common::utils::not_found_if_none( @@ -1292,27 +1392,37 @@ pub async fn get_result_by_id( let value = match result_id { JobResult::ListJob(x) => { - let rows = sqlx::query_scalar!( + let rows = sqlx::query( "SELECT result FROM completed_job WHERE id = ANY($1) AND workspace_id = $2", - x.as_slice(), - w_id, ) + .bind(x.as_slice()) + .bind(w_id) .fetch_all(&db) .await? .into_iter() - .filter_map(|x| x) - .collect::>(); - serde_json::json!(rows) + .filter_map(|x| ResultR::from_row(&x).ok().and_then(|x| x.result)) + .collect::>>>(); + to_raw_value(&rows) } - JobResult::SingleJob(x) => sqlx::query_scalar!( - "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2", - x, - w_id, + JobResult::SingleJob(x) => sqlx::query( + "SELECT result #> $3 as result FROM completed_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(x) + .bind(w_id) + .bind( + json_path + .map(|x| x.split(".").map(|x| x.to_string()).collect::>()) + .unwrap_or_default(), ) .fetch_optional(&db) .await? + .map(|r| { + ResultR::from_row(&r) + .ok() + .and_then(|x| x.result.map(|x| x.0)) + }) .flatten() - .unwrap_or(serde_json::Value::Null), + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)), }; Ok(value) @@ -1366,7 +1476,7 @@ pub async fn get_queued_job<'c>( w_id: &str, tx: &mut Transaction<'c, Postgres>, ) -> error::Result> { - let r = sqlx::query_as::<_, QueuedJob>( + let r = sqlx::query( "SELECT * FROM queue WHERE id = $1 AND workspace_id = $2", ) @@ -1374,7 +1484,11 @@ pub async fn get_queued_job<'c>( .bind(w_id) .fetch_optional(&mut **tx) .await?; - Ok(r) + if let Some(row) = r { + Ok(Some(QueuedJob::from_row(&row)?.to_owned())) + } else { + Ok(None) + } } pub enum PushIsolationLevel<'c, R: rsmq_async::RsmqConnection + Send + 'c> { @@ -1407,13 +1521,144 @@ macro_rules! fetch_scalar_isolated { }; } +use sqlx::types::JsonRawValue; + +#[derive(Serialize)] +pub struct PushArgs { + #[serde(flatten)] + pub extra: HashMap>, + #[serde(flatten)] + pub args: Json, +} + +#[axum::async_trait] +impl FromRequest for PushArgs>> +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request( + req: Request, + _state: &S, + ) -> Result { + let (content_type, mut extra, use_raw) = { + let headers_map = req.headers(); + let content_type_header = headers_map.get(CONTENT_TYPE); + let content_type = content_type_header.and_then(|value| value.to_str().ok()); + ( + content_type, + build_extra(&headers_map), + req.uri().query().is_some_and(|x| x.contains("raw=true")), + ) + }; + + if content_type.is_none() || content_type.unwrap().starts_with("application/json") { + let bytes = Bytes::from_request(req, _state) + .await + .map_err(IntoResponse::into_response)?; + let str = String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; + + if use_raw { + extra.insert("raw_string".to_string(), to_raw_value(&str)); + } + + let wrap_body = str.len() > 0 && str.chars().next().unwrap() != '{'; + + if wrap_body { + let args = serde_json::from_str::>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())? + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + let mut hm = HashMap::new(); + hm.insert("body".to_string(), args); + Ok(PushArgs { extra, args: Json(hm) }) + } else { + let hm = serde_json::from_str::>>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())? + .unwrap_or_else(HashMap::new); + Ok(PushArgs { extra, args: Json(hm) }) + } + } else if content_type + .unwrap() + .starts_with("application/x-www-form-urlencoded") + { + let Form(payload): Form>>> = + req.extract().await.map_err(IntoResponse::into_response)?; + return Ok(PushArgs { + extra: HashMap::new(), + args: Json(payload.unwrap_or_else(HashMap::new)), + }); + } else { + Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response()) + } + } +} + +lazy_static::lazy_static! { + static ref INCLUDE_HEADERS: Vec = std::env::var("INCLUDE_HEADERS") + .ok().map(|x| x + .split(',') + .map(|s| s.to_string()) + .collect()).unwrap_or_default(); +} + +pub fn build_extra(headers: &HeaderMap) -> HashMap> { + let mut args = HashMap::new(); + let whitelist = headers + .get("include_header") + .map(|s| { + s.to_str() + .unwrap_or_default() + .split(",") + .map(|s| s.to_string()) + .collect::>() + }) + .unwrap_or_default(); + + whitelist + .iter() + .chain(INCLUDE_HEADERS.iter()) + .for_each(|h| { + if let Some(v) = headers.get(h) { + args.insert( + h.to_string().to_lowercase().replace('-', "_"), + to_raw_value(&v.to_str().unwrap().to_string()), + ); + } + }); + args +} + +impl PushArgs>> { + pub fn empty() -> Self { + PushArgs { extra: HashMap::new(), args: Json(HashMap::new()) } + } +} + +pub fn empty_args() -> Box { + return JsonRawValue::from_string("{}".to_string()).unwrap(); +} + +impl From>> for PushArgs>> { + fn from(value: HashMap>) -> Self { + PushArgs { extra: HashMap::new(), args: Json(value) } + } +} + +// impl From> for PushArgs { +// fn from(value: PushArgsInner) -> Self { +// PushArgs::Unwrapped(value) +// } +// } + // #[instrument(level = "trace", skip_all)] -pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( +pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection + Send + 'c>( _db: &Pool, mut tx: PushIsolationLevel<'c, R>, workspace_id: &str, job_payload: JobPayload, - args: serde_json::Map, + args: T, user: &str, email: &str, permissioned_as: String, @@ -1430,8 +1675,6 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( custom_timeout: Option, flow_step_id: Option, ) -> Result<(Uuid, QueueTransaction<'c, R>), Error> { - let args_json = serde_json::Value::Object(args); - #[cfg(feature = "enterprise")] { let premium_workspace = *CLOUD_HOSTED @@ -1835,7 +2078,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( script_path.clone(), raw_code, raw_lock, - args_json, + Json(args) as Json, job_kind.clone() as JobKind, schedule_path, raw_flow.map(|f| serde_json::json!(f)), @@ -1852,7 +2095,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( concurrency_time_window_s, custom_timeout, flow_step_id, - cache_ttl + cache_ttl, ) .fetch_one(&mut tx) .await diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 8eab742dec..d54158d056 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -23,7 +23,6 @@ windmill-common = { workspace = true, features = [ "prometheus", "tracing_init", ] } -windmill-api-client.workspace = true windmill-parser.workspace = true windmill-parser-ts.workspace = true windmill-parser-go.workspace = true @@ -73,6 +72,7 @@ pem = { workspace = true, optional = true } urlencoding.workspace = true nix.workspace = true bytes.workspace = true +reqwest.workspace = true [build-dependencies] deno_fetch.workspace = true diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 7838f648ec..41298e8d52 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -2,17 +2,18 @@ use std::{collections::HashMap, process::Stdio}; use itertools::Itertools; use regex::Regex; -use serde_json::{json, Value}; +use serde_json::{json, value::RawValue}; +use sqlx::types::Json; use tokio::process::Command; -use windmill_common::{error::Error, jobs::QueuedJob}; +use windmill_common::{error::Error, jobs::QueuedJob, worker::to_raw_value}; const BIN_BASH: &str = "/bin/bash"; const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto"); use crate::{ common::{ - get_reserved_variables, handle_child, read_file, read_file_content, set_logs, - start_child_process, transform_json_value, write_file, + build_args_map, get_reserved_variables, handle_child, read_file, read_file_content, + set_logs, start_child_process, write_file, }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV, @@ -26,6 +27,7 @@ lazy_static::lazy_static! { #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_bash_job( logs: &mut String, + mem_peak: &mut i32, job: &QueuedJob, db: &sqlx::Pool, client: &AuthedClientBackgroundTask, @@ -35,7 +37,7 @@ pub async fn handle_bash_job( base_internal_url: &str, worker_name: &str, envs: HashMap, -) -> Result { +) -> Result, Error> { logs.push_str("\n\n--- BASH CODE EXECUTION ---\n"); set_logs(logs, &job.id, db).await; write_file( @@ -48,29 +50,19 @@ pub async fn handle_bash_job( let mut reserved_variables = get_reserved_variables(job, &token, db).await?; reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); - let client = client.get_authed().await; - let hm = match transform_json_value( - "args", - &client, - &job.workspace_id, - job.args.clone().unwrap_or_else(|| json!({})), - job, - db, - ) - .await? - { - Value::Object(ref hm) => hm.clone(), - _ => serde_json::Map::new(), + let args = build_args_map(job, client, db).await?.map(Json); + let job_args = if args.is_some() { + args.as_ref() + } else { + job.args.as_ref() }; + let args_owned = windmill_parser_bash::parse_bash_sig(&content)? .args .iter() .map(|arg| { - hm.get(&arg.name) - .and_then(|v| match v { - Value::String(s) => Some(s.clone()), - _ => serde_json::to_string(v).ok(), - }) + job_args + .and_then(|x| x.get(&arg.name).map(|x| raw_to_string(x.get()))) .unwrap_or_else(String::new) }) .collect::>(); @@ -122,6 +114,7 @@ pub async fn handle_bash_job( &job.id, db, logs, + mem_peak, child, !*DISABLE_NSJAIL, worker_name, @@ -143,7 +136,7 @@ pub async fn handle_bash_job( if let Ok(metadata) = tokio::fs::metadata(&result_out_path).await { if metadata.len() > 0 { let result = read_file_content(&result_out_path).await?; - return Ok(json!(result)); + return Ok(to_raw_value(&json!(result))); } } @@ -153,12 +146,20 @@ pub async fn handle_bash_job( .last() .map(|x| ANSI_ESCAPE_RE.replace_all(x, "").to_string()) .unwrap_or_else(String::new)); - Ok(last_line) + Ok(to_raw_value(&last_line)) } +fn raw_to_string(x: &str) -> String { + match serde_json::from_str::(x) { + Ok(serde_json::Value::String(x)) => x, + Ok(x) => serde_json::to_string(&x).unwrap_or_else(|_| String::new()), + _ => String::new(), + } +} #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_powershell_job( logs: &mut String, + mem_peak: &mut i32, job: &QueuedJob, db: &sqlx::Pool, client: &AuthedClientBackgroundTask, @@ -168,36 +169,25 @@ pub async fn handle_powershell_job( base_internal_url: &str, worker_name: &str, envs: HashMap, -) -> Result { +) -> Result, Error> { logs.push_str("\n\n--- POWERSHELL CODE EXECUTION ---\n"); set_logs(logs, &job.id, db).await; let pwsh_args = { - let client = client.get_authed().await; - let hm = match transform_json_value( - "args", - &client, - &job.workspace_id, - job.args.clone().unwrap_or_else(|| json!({})), - job, - db, - ) - .await? - { - Value::Object(ref hm) => hm.clone(), - _ => serde_json::Map::new(), + let args = build_args_map(job, client, db).await?.map(Json); + let job_args = if args.is_some() { + args.as_ref() + } else { + job.args.as_ref() }; - let args_owned = windmill_parser_bash::parse_powershell_sig(&content)? + let args_owned = windmill_parser_bash::parse_bash_sig(&content)? .args .iter() .map(|arg| { ( arg.name.clone(), - hm.get(&arg.name) - .and_then(|v| match v { - Value::String(s) => Some(s.clone()), - _ => serde_json::to_string(v).ok(), - }) + job_args + .and_then(|x| x.get(&arg.name).map(|x| raw_to_string(x.get()))) .unwrap_or_else(String::new), ) }) @@ -262,6 +252,7 @@ pub async fn handle_powershell_job( &job.id, db, logs, + mem_peak, child, !*DISABLE_NSJAIL, worker_name, @@ -277,5 +268,5 @@ pub async fn handle_powershell_job( .last() .map(|x| ANSI_ESCAPE_RE.replace_all(x, "").to_string()) .unwrap_or_else(String::new)); - Ok(last_line) + Ok(to_raw_value(&last_line)) } diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index cc2570cfcc..ccc5bf04ef 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -1,12 +1,12 @@ -use serde_json::{json, Value}; -use windmill_common::error::Error; +use serde_json::{json, value::RawValue, Value}; use windmill_common::jobs::QueuedJob; +use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::parse_bigquery_sig; use windmill_queue::HTTP_CLIENT; use serde::Deserialize; -use crate::{common::transform_json_value, AuthedClient}; +use crate::{common::build_args_values, AuthedClientBackgroundTask}; use gcp_auth::{AuthenticationManager, CustomServiceAccount}; @@ -52,19 +52,12 @@ struct BigqueryError { } pub async fn do_bigquery( - job: QueuedJob, - client: &AuthedClient, + job: &QueuedJob, + client: &AuthedClientBackgroundTask, query: &str, db: &sqlx::Pool, -) -> windmill_common::error::Result { - let args = if let Some(args) = &job.args { - Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?) - } else { - None - }; - - let bigquery_args: Value = serde_json::from_value(args.unwrap_or_else(|| json!({}))) - .map_err(|e| Error::ExecutionErr(e.to_string()))?; +) -> windmill_common::error::Result> { + let bigquery_args = build_args_values(job, client, db).await?; let database = bigquery_args .get("database") @@ -85,14 +78,6 @@ pub async fn do_bigquery( .await .map_err(|e| Error::ExecutionErr(e.to_string()))?; - let args = &job - .args - .clone() - .unwrap_or_else(|| json!({})) - .as_object() - .map(|x| x.to_owned()) - .unwrap_or_else(|| json!({}).as_object().unwrap().to_owned()); - let mut statement_values: Vec = vec![]; let sig = parse_bigquery_sig(&query) @@ -102,7 +87,7 @@ pub async fn do_bigquery( for arg in &sig { let arg_t = arg.otyp.clone().unwrap_or_else(|| "string".to_string()); let arg_n = arg.clone().name; - let arg_v = args.get(&arg.name).cloned().unwrap_or(json!("")); + let arg_v = bigquery_args.get(&arg.name).cloned().unwrap_or(json!("")); let bigquery_v = if arg_t.ends_with("[]") { let base_type = arg_t.strip_suffix("[]").unwrap_or(&arg_t); json!({ @@ -114,7 +99,7 @@ pub async fn do_bigquery( } }, "parameterValue": { - "arrayValues": args + "arrayValues": bigquery_args .get(&arg.name) .unwrap_or(&json!([])) .as_array() @@ -177,7 +162,7 @@ pub async fn do_bigquery( } if result.rows.is_none() || result.rows.as_ref().unwrap().len() == 0 { - return Ok(Value::Array(vec![])); + return Ok(serde_json::from_str("[]").unwrap()); } if result.schema.is_none() { @@ -217,9 +202,9 @@ pub async fn do_bigquery( }); Value::from(row_map) }) - .collect(); + .collect::>(); - return Ok(rows); + return Ok(to_raw_value(&rows)); } Err(e) => match response.json::().await { Ok(bq_err) => return Err(Error::ExecutionErr(bq_err.error.message)), diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 79aa96e059..2940398ea8 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -9,6 +9,7 @@ use anyhow::Context; use base64::Engine; use itertools::Itertools; use regex::Regex; +use serde_json::value::RawValue; use uuid::Uuid; #[cfg(feature = "enterprise")] @@ -63,6 +64,7 @@ lazy_static::lazy_static! { pub async fn gen_lockfile( logs: &mut String, + mem_peak: &mut i32, job_id: &Uuid, w_id: &str, db: &sqlx::Pool, @@ -109,6 +111,7 @@ pub async fn gen_lockfile( job_id, db, logs, + mem_peak, child_process, false, worker_name, @@ -152,6 +155,7 @@ pub async fn gen_lockfile( install_lockfile( logs, + mem_peak, job_id, w_id, db, @@ -187,6 +191,7 @@ pub async fn gen_lockfile( pub async fn install_lockfile( logs: &mut String, + mem_peak: &mut i32, job_id: &Uuid, w_id: &str, db: &sqlx::Pool, @@ -208,6 +213,7 @@ pub async fn install_lockfile( job_id, db, logs, + mem_peak, child_process, false, worker_name, @@ -241,6 +247,7 @@ pub fn get_trusted_deps(code: &str) -> Vec { pub async fn handle_bun_job( requirements_o: Option, logs: &mut String, + mem_peak: &mut i32, job: &QueuedJob, db: &sqlx::Pool, client: &AuthedClientBackgroundTask, @@ -250,7 +257,7 @@ pub async fn handle_bun_job( worker_name: &str, envs: HashMap, shared_mount: &str, -) -> error::Result { +) -> error::Result> { let _ = write_file(job_dir, "main.ts", inner_content).await?; let common_bun_proc_envs: HashMap = @@ -283,6 +290,7 @@ pub async fn handle_bun_job( install_lockfile( logs, + mem_peak, &job.id, &job.workspace_id, db, @@ -305,6 +313,7 @@ pub async fn handle_bun_job( set_logs(&logs, &job.id, &db).await; let _ = gen_lockfile( logs, + mem_peak, &job.id, &job.workspace_id, db, @@ -385,12 +394,12 @@ run().catch(async (e) => {{ }; let reserved_variables_args_out_f = async { - let client = client.get_authed().await; let args_and_out_f = async { create_args_and_out_file(&client, job, job_dir, db).await?; Ok(()) as Result<()> }; let reserved_variables_f = async { + let client = client.get_authed().await; let vars = get_reserved_variables(job, &client.token, db).await?; Ok(vars) as Result> }; @@ -491,6 +500,7 @@ plugin(p) &job.id, db, logs, + mem_peak, child, false, worker_name, @@ -528,6 +538,9 @@ pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap, @@ -541,7 +554,7 @@ pub async fn start_worker( script_path: &str, token: &str, job_completed_tx: Sender, - mut jobs_rx: Receiver, + mut jobs_rx: Receiver>, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> Result<()> { use std::task::Poll; @@ -549,6 +562,7 @@ pub async fn start_worker( use futures::{future, Future}; let mut logs = "".to_string(); + let mut mem_peak: i32 = 0; let _ = write_file(job_dir, "main.ts", inner_content).await?; let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(&base_internal_url).await; @@ -591,6 +605,7 @@ pub async fn start_worker( } install_lockfile( &mut logs, + &mut mem_peak, &Uuid::nil(), &w_id, db, @@ -604,6 +619,7 @@ pub async fn start_worker( logs.push_str("\n\n--- BUN INSTALL ---\n"); let _ = gen_lockfile( &mut logs, + &mut mem_peak, &Uuid::nil(), &w_id, db, @@ -807,8 +823,8 @@ plugin(p) tracing::debug!("processed job"); let result = serde_json::from_str(&line).expect("json is ok"); - let job: QueuedJob = jobs.pop_front().expect("pop"); - job_completed_tx.send(JobCompleted { job , result, logs: "".to_string(), success: true, cached_res_path: None, token: token.to_string() }).await.unwrap(); + let job: Arc = jobs.pop_front().expect("pop"); + job_completed_tx.send(JobCompleted { job , result, logs: "".to_string(), mem_peak: 0, success: true, cached_res_path: None, token: token.to_string() }).await.unwrap(); } else { tracing::info!("dedicated worker process exited"); break; @@ -820,7 +836,7 @@ plugin(p) tracing::debug!("received job"); jobs.push_back(job.clone()); // write_stdin(&mut stdin, &serde_json::to_string(&job.args.unwrap_or_else(|| serde_json::json!({"x": job.id}))).expect("serialize")).await?; - write_stdin(&mut stdin, &serde_json::to_string(&job.args.unwrap_or_else(|| serde_json::json!({}))).expect("serialize")).await?; + write_stdin(&mut stdin, &serde_json::to_string(&job.args).expect("serialize")).await?; stdin.flush().await.context("stdin flush")?; } else { tracing::debug!("job channel closed"); diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index d3ce1fe3ac..13d2dce1b6 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1,11 +1,14 @@ use async_recursion::async_recursion; +use itertools::Itertools; use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; +use regex::Regex; +use serde::Serialize; +use serde_json::value::RawValue; use serde_json::{json, Value}; use sqlx::{Pool, Postgres}; use tokio::process::Command; use tokio::{fs::File, io::AsyncReadExt}; -use windmill_api_client::{types::CreateResource, Client}; use windmill_common::worker::CLOUD_HOSTED; use windmill_common::{ error::{self, Error}, @@ -41,24 +44,60 @@ use futures::{ }; use crate::{ - AuthedClient, MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGTERM, ROOT_CACHE_DIR, TIMEOUT_DURATION, - WHITELIST_ENVS, + AuthedClient, AuthedClientBackgroundTask, MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGTERM, + ROOT_CACHE_DIR, TIMEOUT_DURATION, WHITELIST_ENVS, }; +pub async fn build_args_map<'a>( + job: &'a QueuedJob, + client: &AuthedClientBackgroundTask, + db: &Pool, +) -> error::Result>>> { + if let Some(args) = &job.args { + return transform_json(client, &job.workspace_id, &args.0, &job, db).await; + } + return Ok(None); +} + +pub async fn build_args_values( + job: &QueuedJob, + client: &AuthedClientBackgroundTask, + db: &Pool, +) -> error::Result> { + if let Some(args) = &job.args { + transform_json_as_values(client, &job.workspace_id, &args.0, &job, db).await + } else { + Ok(HashMap::new()) + } +} + #[tracing::instrument(level = "trace", skip_all)] pub async fn create_args_and_out_file( - client: &AuthedClient, + client: &AuthedClientBackgroundTask, job: &QueuedJob, job_dir: &str, db: &Pool, ) -> Result<(), Error> { - let args = if let Some(args) = &job.args { - Some(transform_json_value("args", client, &job.workspace_id, args.clone(), job, db).await?) + if let Some(args) = &job.args { + if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, db).await? { + write_file( + job_dir, + "args.json", + &serde_json::to_string(&x).unwrap_or_else(|_| "{}".to_string()), + ) + .await?; + } else { + write_file( + job_dir, + "args.json", + &serde_json::to_string(&args).unwrap_or_else(|_| "{}".to_string()), + ) + .await?; + } } else { - None + write_file(job_dir, "args.json", "{}").await?; }; - let ser_args = serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?; - write_file(job_dir, "args.json", &ser_args).await?; + write_file(job_dir, "result.json", "").await?; Ok(()) } @@ -81,6 +120,80 @@ pub async fn write_file_binary(dir: &str, path: &str, content: &[u8]) -> error:: Ok(file) } +lazy_static::lazy_static! { + static ref RE_RES_VAR: Regex = Regex::new(r#"\$(?:var|res)\:"#).unwrap(); +} + +pub async fn transform_json<'a>( + client: &AuthedClientBackgroundTask, + workspace: &str, + vs: &'a HashMap>, + job: &QueuedJob, + db: &Pool, +) -> error::Result>>> { + let mut has_match = false; + for (_, v) in vs { + let inner_vs = v.get(); + if (*RE_RES_VAR).is_match(inner_vs) { + has_match = true; + break; + } + } + if !has_match { + return Ok(None); + } + let mut r = HashMap::new(); + for (k, v) in vs { + let inner_vs = v.get(); + if (*RE_RES_VAR).is_match(inner_vs) { + let value = serde_json::from_str(inner_vs).map_err(|e| { + error::Error::InternalErr(format!("Error while parsing inner arg: {e}")) + })?; + let transformed = + transform_json_value(&k, &client.get_authed().await, workspace, value, job, db) + .await?; + let as_raw = serde_json::from_value(transformed).map_err(|e| { + error::Error::InternalErr(format!("Error while parsing inner arg: {e}")) + })?; + r.insert(k.to_string(), as_raw); + } else { + r.insert(k.to_string(), v.to_owned()); + } + } + Ok(Some(r)) +} + +pub async fn transform_json_as_values<'a>( + client: &AuthedClientBackgroundTask, + workspace: &str, + vs: &'a HashMap>, + job: &QueuedJob, + db: &Pool, +) -> error::Result> { + let mut r: HashMap = HashMap::new(); + for (k, v) in vs { + let inner_vs = v.get(); + if (*RE_RES_VAR).is_match(inner_vs) { + let value = serde_json::from_str(inner_vs).map_err(|e| { + error::Error::InternalErr(format!("Error while parsing inner arg: {e}")) + })?; + let transformed = + transform_json_value(&k, &client.get_authed().await, workspace, value, job, db) + .await?; + let as_raw = serde_json::from_value(transformed).map_err(|e| { + error::Error::InternalErr(format!("Error while parsing inner arg: {e}")) + })?; + r.insert(k.to_string(), as_raw); + } else { + r.insert( + k.to_string(), + serde_json::from_str(v.get()).unwrap_or_else(|_| serde_json::Value::Null), + ); + } + } + Ok(r) +} + #[async_recursion] pub async fn transform_json_value( name: &str, @@ -94,11 +207,10 @@ pub async fn transform_json_value( Value::String(y) if y.starts_with("$var:") => { let path = y.strip_prefix("$var:").unwrap(); client - .get_client() - .get_variable_value(workspace, path) + .get_variable_value(path) .await + .map(|x| json!(x)) .map_err(|_| Error::NotFound(format!("Variable {path} not found for `{name}`"))) - .map(|v| json!(v.into_inner())) } Value::String(y) if y.starts_with("$res:") => { let path = y.strip_prefix("$res:").unwrap(); @@ -107,12 +219,13 @@ pub async fn transform_json_value( "Argument `{name}` is an invalid resource path: {path}", ))); } - Ok(client - .get_client() - .get_resource_value_interpolated(workspace, path, Some(&job.id)) + client + .get_resource_value_interpolated::( + path, + Some(job.id.to_string()), + ) .await - .map_err(|_| Error::NotFound(format!("Resource {path} not found for `{name}`")))? - .into_inner()) + .map_err(|_| Error::NotFound(format!("Resource {path} not found for `{name}`"))) } Value::String(y) if y.starts_with("$") => { let flow_path = if let Some(uuid) = job.parent_job { @@ -167,30 +280,31 @@ pub async fn read_file_content(path: &str) -> error::Result { file.read_to_string(&mut content).await?; Ok(content) } -pub async fn read_file(path: &str) -> error::Result { - // tracing::error!("START1"); - // let start = Instant::now(); - let r = if *CLOUD_HOSTED { - let content = read_file_content(path).await?; - if content.len() > MAX_RESULT_SIZE { - return Err(error::Error::ExecutionErr("Result is too large for the cloud app (limit 2MB). - If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned())); - }; - serde_json::from_str(&content) - .map_err(|e| error::Error::ExecutionErr(format!("Error parsing result: {e}"))) - } else { - let file = std::fs::File::open(path) - .map_err(|e| error::Error::ExecutionErr(format!("Error opening file {path}: {e}")))?; - let reader = std::io::BufReader::new(file); - - serde_json::from_reader(reader) - .map_err(|e| error::Error::ExecutionErr(format!("Error parsing result: {e}"))) - }; - // tracing::error!("{:?}", start.elapsed()); - return r; +pub async fn read_file_bytes(path: &str) -> error::Result> { + let mut file = File::open(path).await?; + let mut content = Vec::new(); + file.read_to_end(&mut content).await?; + Ok(content) } -pub async fn read_result(job_dir: &str) -> error::Result { + +//this skips more steps than from_str at the cost of being unsafe. The source must ALWAUS gemerate valid json or this can cause UB in the worst case +pub fn unsafe_raw(json: String) -> Box { + unsafe { std::mem::transmute::, Box>(json.into()) } +} + +pub async fn read_file(path: &str) -> error::Result> { + let content = read_file_content(path).await?; + + if *CLOUD_HOSTED && content.len() > MAX_RESULT_SIZE { + return Err(error::Error::ExecutionErr("Result is too large for the cloud app (limit 2MB). + If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned())); + }; + + let r = unsafe_raw(content); + return Ok(r); +} +pub async fn read_result(job_dir: &str) -> error::Result> { return read_file(&format!("{job_dir}/result.json")).await; } @@ -302,6 +416,7 @@ pub async fn handle_child( job_id: &Uuid, db: &Pool, logs: &mut String, + mem_peak: &mut i32, mut child: Child, nsjail: bool, worker_name: &str, @@ -343,6 +458,7 @@ pub async fn handle_child( interval.set_missed_tick_behavior(MissedTickBehavior::Skip); let mut i = 0; + loop { tokio::select!( _ = rx.recv() => break, @@ -358,10 +474,12 @@ pub async fn handle_child( .await .expect("update worker ping"); } - let mem_peak = get_mem_peak(pid, nsjail).await; - tracing::info!("{job_id} in {} still running. mem peak: {}kB", _w_id, mem_peak); - let mem_peak = if mem_peak > 0 { Some(mem_peak) } else { None }; - if sqlx::query_scalar!("UPDATE queue SET mem_peak = GREATEST($1, mem_peak), last_ping = now() WHERE id = $2 RETURNING canceled", mem_peak, job_id) + let current_mem = get_mem_peak(pid, nsjail).await; + if current_mem > *mem_peak { + *mem_peak = current_mem + } + tracing::info!("{job_id} in {_w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB"); + if sqlx::query_scalar!("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled", *mem_peak, job_id) .fetch_optional(&db) .await .map(|v| Some(true) == v) @@ -570,7 +688,7 @@ pub async fn handle_child( let (wait_result, _) = tokio::join!(wait_on_child, lines); - tracing::info!(%job_id, "child process '{child_name}' for {job_id} took {}ms", start.elapsed().as_millis()); + tracing::info!(%job_id, "child process '{child_name}' for {job_id} took {}ms, mem_peak: {:?}", start.elapsed().as_millis(), mem_peak); match wait_result { _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!( "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)" @@ -657,27 +775,49 @@ fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) { } } -pub fn hash_args(v: &serde_json::Value) -> i64 { - let mut dh = DefaultHasher::new(); - serde_json::to_string(v).unwrap().hash(&mut dh); - dh.finish() as i64 +pub fn hash_args(v: &Option>>>) -> i64 { + if let Some(vs) = v { + let mut dh = DefaultHasher::new(); + let hm = &vs.0; + for k in hm.keys().sorted() { + k.hash(&mut dh); + hm.get(k).unwrap().get().hash(&mut dh); + } + dh.finish() as i64 + } else { + 0 + } } -pub async fn save_in_cache(client: &AuthedClient, job: &QueuedJob, cached_path: String, r: &Value) { - let client: &Client = client.get_client(); +#[derive(Serialize)] +struct StoreCachedResource<'a> { + expire: i64, + value: &'a RawValue, +} + +pub async fn save_in_cache<'a>( + db: &Pool, + job: &QueuedJob, + cached_path: String, + r: &'a RawValue, +) { let expire = chrono::Utc::now().timestamp() + job.cache_ttl.unwrap() as i64; - let cr = &CreateResource { - path: cached_path, - description: None, - resource_type: "cache".to_string(), - value: serde_json::json!({ - "value": r, - "expire": expire - }), - }; - if let Err(e) = client - .create_resource(&job.workspace_id, Some(true), cr) - .await + + let store_cache_resource = StoreCachedResource { expire, value: r }; + let raw_json = sqlx::types::Json(store_cache_resource); + + if let Err(e) = sqlx::query!( + "INSERT INTO resource + (workspace_id, path, value, resource_type) + VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path) + DO UPDATE SET value = $3", + job.workspace_id, + cached_path, + raw_json as sqlx::types::Json, + "cache" + ) + .execute(db) + .await { tracing::error!("Error creating cache resource {e}") } diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 5348e2eca0..dc45275d3b 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -1,6 +1,7 @@ use std::{collections::HashMap, process::Stdio}; use itertools::Itertools; +use serde_json::value::RawValue; use uuid::Uuid; use crate::{ @@ -73,6 +74,7 @@ pub async fn generate_deno_lock( job_id: &Uuid, code: &str, logs: &mut String, + mem_peak: &mut i32, job_dir: &str, db: &sqlx::Pool, w_id: &str, @@ -117,6 +119,7 @@ pub async fn generate_deno_lock( job_id, db, logs, + mem_peak, child_process, false, worker_name, @@ -138,6 +141,7 @@ pub async fn generate_deno_lock( pub async fn handle_deno_job( requirements_o: Option, logs: &mut String, + mem_peak: &mut i32, job: &QueuedJob, db: &sqlx::Pool, client: &AuthedClientBackgroundTask, @@ -146,7 +150,7 @@ pub async fn handle_deno_job( base_internal_url: &str, worker_name: &str, envs: HashMap, -) -> error::Result { +) -> error::Result> { // let mut start = Instant::now(); logs.push_str("\n\n--- DENO CODE EXECUTION ---\n"); @@ -247,18 +251,18 @@ run().catch(async (e) => {{ }; let reserved_variables_args_out_f = async { - let client = client.get_authed().await; let args_and_out_f = async { create_args_and_out_file(&client, job, job_dir, db).await?; Ok(()) as Result<()> }; let reserved_variables_f = async { + let client = client.get_authed().await; let mut vars = get_reserved_variables(job, &client.token, db).await?; vars.insert("RUST_LOG".to_string(), "info".to_string()); - Ok(vars) as Result> + Ok((vars, client.token)) as Result<(HashMap, String)> }; let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?; - Ok((reserved_variables, client.token)) as error::Result<(HashMap, String)> + Ok(reserved_variables) as error::Result<(HashMap, String)> }; let (_, (reserved_variables, token), _, _, _) = tokio::try_join!( @@ -325,6 +329,7 @@ run().catch(async (e) => {{ &job.id, db, logs, + mem_peak, child, false, worker_name, diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 11ff53892e..242eed0ce1 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -1,6 +1,7 @@ use std::{collections::HashMap, process::Stdio}; use itertools::Itertools; +use serde_json::value::RawValue; use tokio::{ fs::{DirBuilder, File}, io::AsyncReadExt, @@ -33,6 +34,7 @@ lazy_static::lazy_static! { #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_go_job( logs: &mut String, + mem_peak: &mut i32, job: &QueuedJob, db: &sqlx::Pool, client: &AuthedClientBackgroundTask, @@ -43,7 +45,7 @@ pub async fn handle_go_job( base_internal_url: &str, worker_name: &str, envs: HashMap, -) -> Result { +) -> Result, Error> { //go does not like executing modules at temp root let job_dir = &format!("{job_dir}/go"); let bin_path = if let Some(requirements) = requirements_o.clone() { @@ -68,8 +70,6 @@ pub async fn handle_go_job( (false, false) }; - let client = &client.get_authed().await; - if !bin_exists { logs.push_str("\n\n--- GO DEPENDENCIES SETUP ---\n"); set_logs(logs, &job.id, db).await; @@ -78,6 +78,7 @@ pub async fn handle_go_job( &job.id, inner_content, logs, + mem_peak, job_dir, db, true, @@ -193,6 +194,7 @@ func Run(req Req) (interface{{}}, error){{ &job.id, db, logs, + mem_peak, build_go_process, false, worker_name, @@ -216,6 +218,8 @@ func Run(req Req) (interface{{}}, error){{ create_args_and_out_file(client, job, job_dir, db).await?; } + let client = &client.get_authed().await; + let reserved_variables = get_reserved_variables(job, &client.token, db).await?; let child = if !*DISABLE_NSJAIL { @@ -270,6 +274,7 @@ func Run(req Req) (interface{{}}, error){{ &job.id, db, logs, + mem_peak, child, !*DISABLE_NSJAIL, worker_name, @@ -307,6 +312,7 @@ pub async fn install_go_dependencies( job_id: &Uuid, code: &str, logs: &mut String, + mem_peak: &mut i32, job_dir: &str, db: &sqlx::Pool, non_dep_job: bool, @@ -329,6 +335,7 @@ pub async fn install_go_dependencies( job_id, db, logs, + mem_peak, child_process, false, worker_name, @@ -392,6 +399,7 @@ pub async fn install_go_dependencies( job_id, db, logs, + mem_peak, child_process, false, worker_name, diff --git a/backend/windmill-worker/src/graphql_executor.rs b/backend/windmill-worker/src/graphql_executor.rs index 7a6b359f82..7c0fb5d6c0 100644 --- a/backend/windmill-worker/src/graphql_executor.rs +++ b/backend/windmill-worker/src/graphql_executor.rs @@ -1,13 +1,14 @@ use std::collections::HashMap; -use serde_json::{json, Value}; +use serde_json::{json, value::RawValue}; +use sqlx::types::Json; use windmill_common::error::Error; use windmill_common::jobs::QueuedJob; use windmill_queue::HTTP_CLIENT; use serde::Deserialize; -use crate::{common::transform_json_value, AuthedClient}; +use crate::{common::build_args_map, AuthedClientBackgroundTask}; #[derive(Deserialize)] struct GraphqlApi { @@ -18,7 +19,7 @@ struct GraphqlApi { #[derive(Deserialize)] struct GraphqlResponse { - data: Option, + data: Option>, errors: Option>, } @@ -28,34 +29,28 @@ struct GraphqlError { } pub async fn do_graphql( - job: QueuedJob, - client: &AuthedClient, + job: &QueuedJob, + client: &AuthedClientBackgroundTask, query: &str, db: &sqlx::Pool, -) -> windmill_common::error::Result { - let args = if let Some(args) = &job.args { - Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?) +) -> windmill_common::error::Result> { + let args = build_args_map(job, client, db).await?.map(Json); + let job_args = if args.is_some() { + args.as_ref() } else { - None + job.args.as_ref() }; - let graphql_args: serde_json::Value = serde_json::from_value(args.unwrap_or_else(|| json!({}))) - .map_err(|e| Error::ExecutionErr(e.to_string()))?; - let api = - serde_json::from_value::(graphql_args.get("api").unwrap_or(&json!({})).clone()) - .map_err(|e: serde_json::Error| Error::ExecutionErr(e.to_string()))?; - - let args = &job - .args - .clone() - .unwrap_or_else(|| json!({})) - .as_object() - .map(|x| x.to_owned()) - .unwrap_or_else(|| json!({}).as_object().unwrap().to_owned()); + let api = if let Some(db) = job_args.as_ref().and_then(|x| x.get("api")) { + serde_json::from_str::(db.get()) + .map_err(|e| Error::ExecutionErr(e.to_string()))? + } else { + return Err(Error::BadRequest("Missing api argument".to_string())); + }; let mut request = HTTP_CLIENT.post(api.base_url).json(&json!({ "query": query, - "variables": args + "variables": job_args })); if let Some(token) = &api.bearer_token { @@ -89,5 +84,7 @@ pub async fn do_graphql( } // And then check that we got back the same string we sent over. - return Ok(result.data.unwrap_or(json!({}))); + return Ok(result + .data + .unwrap_or_else(|| serde_json::from_str("{}").unwrap())); } diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 6e77daab6d..cae7af81d7 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -20,7 +20,8 @@ use deno_web::{BlobStore, TimersPermission}; use itertools::Itertools; use lazy_static::lazy_static; use regex::Regex; -use serde_json::Value; +use serde_json::value::RawValue; +use sqlx::types::Json; use tokio::{ sync::{mpsc, oneshot}, time::timeout, @@ -28,7 +29,7 @@ use tokio::{ use uuid::Uuid; use windmill_common::{error::Error, flow_status::JobResult}; -use crate::AuthedClient; +use crate::{common::unsafe_raw, AuthedClient}; #[derive(Debug, Clone)] pub struct IdContext { @@ -68,12 +69,56 @@ impl TimersPermission for PermissionsContainer { } pub struct OptAuthedClient(Option); + pub async fn eval_timeout( expr: String, - env: Vec<(String, serde_json::Value)>, + transform_context: HashMap>>, + flow_input: Option>>>, authed_client: Option<&AuthedClient>, by_id: Option, -) -> anyhow::Result { +) -> anyhow::Result> { + + let expr = expr.trim().to_string(); + + for (k,v) in transform_context.iter() { + if k == &expr { + return Ok(v.as_ref().clone()) + } + } + + if expr.starts_with("flow_input.") { + if let Some(ref flow_input) = flow_input { + for (k,v) in flow_input.iter() { + if &format!("flow_input.{k}") == &expr { + // tracing::error!("FLOW_INPUT"); + return Ok(v.clone()) + } + } + } + } + + let p_id = by_id.as_ref().map(|x| format!("results.{}", x.previous_id)); + + if p_id.is_some() && transform_context.contains_key("previous_result") && &expr == p_id.as_ref().unwrap() { + // tracing::error!("PREVIOUS_RESULT"); + return Ok(transform_context.get("previous_result").unwrap().as_ref().clone()) + } + + if by_id.is_some() && authed_client.is_some() { + if let Some(x) = RE_FULL.captures(&expr).and_then(|x| x.get(1).map(|y| y.as_str())) { + // tracing::error!("{:?}", x.split(".").collect::>()); + let arr = x.split(".").collect::>(); + let mut iter = arr.iter(); + iter.next(); + if let Some(id) = iter.next() { + let path = iter.join("."); + let query = if path.is_empty() { None } else { Some(path) }; + return authed_client.unwrap().get_result_by_id(&by_id.as_ref().unwrap().flow_job.to_string(), id, query).await; + } + } + } + + let expr2 = expr.clone(); let (sender, mut receiver) = oneshot::channel::(); let has_client = authed_client.is_some(); @@ -81,7 +126,7 @@ pub async fn eval_timeout( timeout( std::time::Duration::from_millis(10000), tokio::task::spawn_blocking(move || { - let mut ops = vec![]; + let mut ops = vec![op_get_context::DECL]; if authed_client.is_some() { ops.extend([ @@ -107,11 +152,35 @@ pub async fn eval_timeout( ..Default::default() }; + + let mut context_keys = transform_context + .keys() + .filter(|x| + expr.contains(&x.to_string()) + ) + .map(|x| x.clone()) + .collect_vec(); + + if !context_keys.contains(&"previous_result".to_string()) && (p_id.is_some() && expr.contains(p_id.as_ref().unwrap())) || expr.contains("error") { + context_keys.push("previous_result".to_string()); + } + let has_flow_input = expr.contains("flow_input"); + if has_flow_input { + context_keys.push("flow_input".to_string()) + } + let mut js_runtime = JsRuntime::new(options); { let op_state = js_runtime.op_state(); let mut op_state = op_state.borrow_mut(); op_state.put(OptAuthedClient(authed_client.clone())); + op_state.put(TransformContext { + flow_input: if has_flow_input { flow_input } else { None }, + envs: transform_context + .into_iter() + .filter(|(a, _)| context_keys.contains(a)) + .collect(), + }) } sender @@ -123,15 +192,21 @@ pub async fn eval_timeout( .build()?; // pretty frail but this it to make the expr more user friendly and not require the user to write await - let expr = ["variable", "step", "resource", "result_by_id"] + let expr = ["variable", "resource"] .into_iter() .fold(expr, replace_with_await); let expr = replace_with_await_result(expr); - let r = runtime.block_on(eval(&mut js_runtime, &expr, env, by_id, has_client))?; + let r = runtime.block_on(eval( + &mut js_runtime, + &expr, + context_keys, + by_id, + has_client, + ))?; - Ok(r) as anyhow::Result + Ok(r) as anyhow::Result> }), ) .await @@ -155,7 +230,9 @@ fn replace_with_await(expr: String, fn_name: &str) -> String { s } lazy_static! { - static ref RE: Regex = Regex::new("(?m)(?Presults.([a-z]|[A-Z]|_|[1-9])+)").unwrap(); + static ref RE: Regex = Regex::new(r"(?m)(?Presults\.(?:[a-z]|[A-Z]|_|[1-9])+)").unwrap(); + static ref RE_FULL: Regex = Regex::new(r"(?m)^results((?:\.(?:(?:[a-z]|[A-Z]|_|[1-9])+))+)$").unwrap(); + } fn replace_with_await_result(expr: String) -> String { @@ -184,10 +261,10 @@ fn add_closing_bracket(s: &str) -> String { async fn eval( context: &mut JsRuntime, expr: &str, - env: Vec<(String, serde_json::Value)>, + transform_context: Vec, by_id: Option, has_client: bool, -) -> anyhow::Result { +) -> anyhow::Result> { let (api_code, by_id_code) = if has_client { let by_id_code = if let Some(by_id) = by_id { format!( @@ -205,12 +282,12 @@ async function result_by_id(node_id) {{ }} }} else {{ let flow_job_id = "{}"; - return await Deno.core.opAsync("op_get_id", [flow_job_id, node_id]); + return JSON.parse(await Deno.core.opAsync("op_get_id", [flow_job_id, node_id])); }} }} async function get_result(id) {{ - return await Deno.core.opAsync("op_get_result", [id]); + return JSON.parse(await Deno.core.opAsync("op_get_result", [id])); }} const results = new Proxy({{}}, {{ get: function(target, name, receiver) {{ @@ -261,24 +338,30 @@ async function resource(path) {{ }; let code = format!( r#" +function get_from_env(name) {{ + return JSON.parse(Deno.core.ops.op_get_context([name])); +}} {api_code} {} +{} {by_id_code} {HAS_CYCLE} ((async () => {{ {f}; -}})()).then((r) => hasCycle(r) ? 'cycle detected' : r) +}})()).then((r) => hasCycle(r) ? 'cycle detected' : r).then(JSON.stringify) "#, - env.into_iter() - .map(|(a, b)| { - format!( - "let {a} = {};\n", - serde_json::to_string(&b) - .unwrap_or_else(|_| "\"error serializing value\"".to_string()) - ) - }) + transform_context + .iter() + .map(|a| { format!("let {a} = get_from_env(\"{a}\");\n",) }) .join(""), + if expr.contains("error") && transform_context.contains(&"previous_result".to_string()) { + "let error = previous_result.error" + } else { + "" + }, ); + + let global = context.execute_script("", code.into())?; let global = context.resolve_value(global).await?; @@ -286,7 +369,8 @@ async function resource(path) {{ let local = v8::Local::new(scope, global); // Deserialize a `v8` object into a Rust type using `serde_v8`, // in this case deserialize to a JSON `Value`. - Ok(serde_v8::from_v8::(scope, local)?) + let r = serde_v8::from_v8::(scope, local)?; + Ok(unsafe_raw(r)) } const HAS_CYCLE: &str = r#" @@ -341,11 +425,7 @@ async fn op_variable( let path = &args[0]; let client = op_state.borrow().borrow::().0.clone(); if let Some(client) = client { - let result = client - .get_client() - .get_variable_value(&client.workspace, path) - .await?; - Ok(result.into_inner()) + Ok(client.get_variable_value(path).await?) } else { anyhow::bail!("No client found in op state"); } @@ -355,16 +435,15 @@ async fn op_variable( async fn op_get_result( op_state: Rc>, args: Vec, -) -> Result { +) -> Result { let id = &args[0]; let client = op_state.borrow().borrow::().0.clone(); if let Some(client) = client { let result = client - .get_client() - .get_completed_job_result(&client.workspace, &id.parse()?) + .get_completed_job_result::>(id, None) .await? .clone(); - Ok(serde_json::json!(result)) + Ok(result.get().to_string()) } else { anyhow::bail!("No client found in op state"); } @@ -374,18 +453,16 @@ async fn op_get_result( async fn op_get_id( op_state: Rc>, args: Vec, -) -> Result, anyhow::Error> { +) -> Result, anyhow::Error> { let flow_job_id = &args[0]; let node_id = &args[1]; let client = op_state.borrow().borrow::().0.clone(); if let Some(client) = client { let result = client - .get_client() - .result_by_id(&client.workspace, flow_job_id, node_id) - .await - .map_or(None, |e| Some(e.into_inner())); - Ok(result) + .get_result_by_id::>>(flow_job_id, node_id, None) + .await?; + Ok(result.map(|x| x.get().to_string())) } else { anyhow::bail!("No client found in op state"); } @@ -400,16 +477,36 @@ async fn op_resource( let client = op_state.borrow().borrow::().0.clone(); if let Some(client) = client { - let result = client - .get_client() - .get_resource_value_interpolated(&client.workspace, path, None) - .await?; - Ok(result.into_inner()) + client.get_resource_value_interpolated(path, None).await } else { anyhow::bail!("No client found in op state"); } } +pub struct TransformContext { + pub envs: HashMap>>, + pub flow_input: Option>>>, +} + +#[op] +fn op_get_context(op_state: Rc>, args: Vec) -> String { + let id = &args[0]; + let ops = op_state.borrow(); + let client = ops.borrow::(); + if id == "flow_input" { + return client + .flow_input + .as_ref() + .and_then(|x| serde_json::to_string(&x).ok()) + .unwrap_or_else(|| "null".to_string()); + } + return client + .envs + .get(id) + .and_then(|x| serde_json::to_string(x).ok()) + .unwrap_or_else(String::new); +} + pub fn transpile_ts(expr: String) -> anyhow::Result { let parsed = deno_ast::parse_module(ParseParams { specifier: "eval.ts".to_string(), @@ -425,7 +522,7 @@ pub fn transpile_ts(expr: String) -> anyhow::Result { static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin")); pub struct MainArgs { - args: Vec, + args: Vec>>, } pub struct LogString { @@ -435,10 +532,14 @@ pub struct LogString { pub async fn eval_fetch_timeout( ts_expr: String, js_expr: String, - args: serde_json::Map, -) -> anyhow::Result<(serde_json::Value, String)> { + args: Option<&Json>>>, +) -> anyhow::Result<(Box, String)> { let (sender, mut receiver) = oneshot::channel::(); let ts_expr2 = ts_expr.clone(); + + let parsed_args = windmill_parser_ts::parse_deno_signature(&ts_expr, true)?.args; + let spread = parsed_args.into_iter().map(|x| args.as_ref().and_then(|args| args.0.get(&x.name).map(|x| x.clone()))).collect::>(); + timeout( std::time::Duration::from_secs(100), tokio::task::spawn_blocking(move || { @@ -497,9 +598,7 @@ pub async fn eval_fetch_timeout( return y*2; }); - let parsed_args = windmill_parser_ts::parse_deno_signature(&ts_expr, true)?.args; - let spread = parsed_args.into_iter().map(|x| args.get(&x.name).map(|x| x.clone()).unwrap_or(serde_json::Value::Null)).collect::>(); - + { let op_state = js_runtime.op_state(); let mut op_state = op_state.borrow_mut(); @@ -527,7 +626,7 @@ pub async fn eval_fetch_timeout( let r = runtime.block_on(future)?; // tracing::info!("total: {:?}", instant.elapsed()); - (r as anyhow::Result).map(|x| (x, js_runtime.op_state().borrow().borrow::().s.clone())) + (r as anyhow::Result>).map(|x| (x, js_runtime.op_state().borrow().borrow::().s.clone())) }), ) .await @@ -541,7 +640,7 @@ pub async fn eval_fetch_timeout( })?? } -async fn eval_fetch(js_runtime: &mut JsRuntime, expr: &str) -> anyhow::Result { +async fn eval_fetch(js_runtime: &mut JsRuntime, expr: &str) -> anyhow::Result> { let _ = js_runtime .load_side_module( &deno_core::resolve_url("file:///eval.ts")?, @@ -552,8 +651,8 @@ async fn eval_fetch(js_runtime: &mut JsRuntime, expr: &str) -> anyhow::Result", r#" -let args = Deno.core.ops.op_get_static_args() -import("file:///eval.ts").then((module) => module.main(...args)) +let args = Deno.core.ops.op_get_static_args().map(JSON.parse) +import("file:///eval.ts").then((module) => module.main(...args)).then(JSON.stringify) "# .to_string() .into(), @@ -564,12 +663,13 @@ import("file:///eval.ts").then((module) => module.main(...args)) let local = v8::Local::new(scope, global); // Deserialize a `v8` object into a Rust type using `serde_v8`, // in this case deserialize to a JSON `Value`. - Ok(serde_v8::from_v8::(scope, local)?) + let r = serde_v8::from_v8::(scope, local)?; + Ok(unsafe_raw(r)) } #[op] -fn op_get_static_args(op_state: Rc>) -> Vec { - return op_state.borrow().borrow::().args.clone(); +fn op_get_static_args(op_state: Rc>) -> Vec> { + return op_state.borrow().borrow::().args.iter().map(|x| x.as_ref().map(|y| y.get().to_string())).collect_vec(); } #[op] @@ -585,21 +685,52 @@ fn op_log(op_state: Rc>, args: Vec) { mod tests { use serde_json::json; + use windmill_common::worker::to_raw_value; // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; #[tokio::test] async fn test_eval() -> anyhow::Result<()> { - let env = vec![ - ("params".to_string(), json!({"test": 2})), - ("value".to_string(), json!({"test": 2})), - ]; + let mut env = HashMap::new(); + env.insert( + "params".to_string(), + Arc::new(to_raw_value(&json!({"test": 2}))), + ); + env.insert( + "value".to_string(), + Arc::new(to_raw_value(&json!({"test": 2}))), + ); + let code = "value.test + params.test"; - let mut runtime = JsRuntime::new(RuntimeOptions::default()); - let res = eval(&mut runtime, code, env, None, false).await?; - assert_eq!(res, json!(4)); + let ops = vec![op_get_context::DECL]; + + let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() }; + let exts = vec![ext]; + + let options = RuntimeOptions { + extensions: exts, + ..Default::default() + }; + + + let mut runtime = JsRuntime::new(options); + { + let op_state = runtime.op_state(); + let mut op_state = op_state.borrow_mut(); + op_state.put(TransformContext { flow_input: None, envs: env.clone() }) + } + + let res = eval( + &mut runtime, + code, + vec!["params".to_string(), "value".to_string()], + None, + false, + ) + .await?; + assert_eq!(res.get(), "4"); Ok(()) } @@ -612,20 +743,33 @@ multiline template`"; let mut runtime = JsRuntime::new(RuntimeOptions::default()); let res = eval(&mut runtime, code, env, None, false).await?; - assert_eq!(res, json!("my 5\nmultiline template")); + assert_eq!(res.get(), "\"my 5\\nmultiline template\""); Ok(()) } #[tokio::test] async fn test_eval_timeout() -> anyhow::Result<()> { - let env = vec![ - ("params".to_string(), json!({"test": 2})), - ("value".to_string(), json!({"test": 2})), - ]; + let mut env = HashMap::new(); + env.insert( + "params".to_string(), + Arc::new(to_raw_value(&json!({"test": 2}))), + ); + env.insert( + "value".to_string(), + Arc::new(to_raw_value(&json!({"test": 2}))), + ); + let code = r#"params.test"#; - let res = eval_timeout(code.to_string(), env, None, None).await?; - assert_eq!(res, json!(2)); + let mut js_runtime = JsRuntime::new(RuntimeOptions::default()); + { + let op_state = js_runtime.op_state(); + let mut op_state = op_state.borrow_mut(); + op_state.put(TransformContext { flow_input: None, envs: env.clone() }) + } + + let res = eval_timeout(code.to_string(), env, None, None, None).await?; + assert_eq!(res.get(), "2"); Ok(()) } @@ -633,9 +777,8 @@ multiline template`"; async fn test_eval_fetch_timeout() -> anyhow::Result<()> { let code = r#"export async function main() { return "" }"#; - let res = - eval_fetch_timeout(code.to_string(), code.to_string(), serde_json::Map::new()).await?; - assert_eq!(res.0, "".to_string()); + let res = eval_fetch_timeout(code.to_string(), code.to_string(), None).await?; + assert_eq!(res.0.get(), "\"\""); Ok(()) } } diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 36d8154a0c..59e5875a5d 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -3,14 +3,15 @@ use mysql_async::{ consts::ColumnType, prelude::*, FromValueError, OptsBuilder, Params, Row, SslOpts, }; use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; +use serde_json::{json, value::RawValue, Value}; +use sqlx::types::Json; use windmill_common::{ error::{to_anyhow, Error}, jobs::QueuedJob, }; use windmill_parser_sql::parse_mysql_sig; -use crate::{common::transform_json_value, AuthedClient}; +use crate::{common::build_args_map, AuthedClientBackgroundTask}; #[derive(Deserialize)] struct MysqlDatabase { @@ -23,23 +24,24 @@ struct MysqlDatabase { } pub async fn do_mysql( - job: QueuedJob, - client: &AuthedClient, + job: &QueuedJob, + client: &AuthedClientBackgroundTask, query: &str, db: &sqlx::Pool, -) -> windmill_common::error::Result { - let args = if let Some(args) = &job.args { - Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?) +) -> windmill_common::error::Result> { + let args = build_args_map(job, client, db).await?.map(Json); + let job_args = if args.is_some() { + args.as_ref() } else { - None + job.args.as_ref() }; - let mysql_args: serde_json::Value = serde_json::from_value(args.unwrap_or_else(|| json!({}))) - .map_err(|e| Error::ExecutionErr(e.to_string()))?; - let database = serde_json::from_value::( - mysql_args.get("database").unwrap_or(&json!({})).clone(), - ) - .map_err(|e: serde_json::Error| Error::ExecutionErr(e.to_string()))?; + let database = if let Some(db) = job_args.and_then(|x| x.get("database")) { + serde_json::from_str::(db.get()) + .map_err(|e| Error::ExecutionErr(e.to_string()))? + } else { + return Err(Error::BadRequest("Missing database argument".to_string())); + }; let opts = OptsBuilder::default() .db_name(Some(database.database)) @@ -61,13 +63,6 @@ pub async fn do_mysql( let pool = mysql_async::Pool::new(opts); let mut conn = pool.get_conn().await.map_err(to_anyhow)?; - let args = &job - .args - .clone() - .unwrap_or_else(|| json!({})) - .as_object() - .map(|x| x.to_owned()) - .unwrap_or_else(|| json!({}).as_object().unwrap().to_owned()); let mut statement_values: Vec = vec![]; let sig = parse_mysql_sig(&query) @@ -76,9 +71,18 @@ pub async fn do_mysql( for arg in &sig { let arg_t = arg.otyp.clone().unwrap_or_else(|| "text".to_string()); - let mysql_v = match args.get(arg.name.as_str()).unwrap_or_else(|| &json!(null)) { + let mysql_v = match job + .args + .as_ref() + .and_then(|x| { + x.get(arg.name.as_str()) + .map(|x| serde_json::from_str::(x.get()).ok()) + }) + .flatten() + .unwrap_or_else(|| json!(null)) + { Value::Null => mysql_async::Value::NULL, - Value::Bool(b) => mysql_async::Value::Int(if *b { 1 } else { 0 }), + Value::Bool(b) => mysql_async::Value::Int(if b { 1 } else { 0 }), Value::String(s) => mysql_async::Value::Bytes(s.as_bytes().to_vec()), Value::Number(n) if n.is_i64() && (arg_t == "int" || arg_t == "integer" || arg_t == "smallint") => @@ -115,7 +119,7 @@ pub async fn do_mysql( pool.disconnect().await.map_err(to_anyhow)?; // And then check that we got back the same string we sent over. - return Ok(json!(rows)); + return Ok(windmill_common::worker::to_raw_value(&json!(rows))); } fn convert_row_to_value(row: Row) -> serde_json::Value { diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 2aa7e2648a..12b499355c 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -5,6 +5,7 @@ use native_tls::{Certificate, TlsConnector}; use postgres_native_tls::MakeTlsConnector; use rust_decimal::{prelude::FromPrimitive, Decimal}; use serde::Deserialize; +use serde_json::value::RawValue; use serde_json::Map; use serde_json::{json, Value}; use tokio_postgres::types::IsNull; @@ -18,11 +19,12 @@ use tokio_postgres::{ }; use uuid::Uuid; use windmill_common::error::{self, Error}; +use windmill_common::worker::to_raw_value; use windmill_common::{error::to_anyhow, jobs::QueuedJob}; use windmill_parser_sql::parse_pgsql_sig; -use crate::common::transform_json_value; -use crate::AuthedClient; +use crate::common::build_args_values; +use crate::AuthedClientBackgroundTask; use bytes::BytesMut; use urlencoding::encode; @@ -38,22 +40,19 @@ struct PgDatabase { } pub async fn do_postgresql( - job: QueuedJob, - client: &AuthedClient, + job: &QueuedJob, + client: &AuthedClientBackgroundTask, query: &str, db: &sqlx::Pool, -) -> error::Result { - let args = if let Some(args) = &job.args { - Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?) - } else { - None - }; +) -> error::Result> { + let pg_args = build_args_values(job, client, db).await?; - let pg_args: serde_json::Value = serde_json::from_value(args.unwrap_or_else(|| json!({}))) - .map_err(|e| Error::ExecutionErr(e.to_string()))?; - let database = - serde_json::from_value::(pg_args.get("database").unwrap_or(&json!({})).clone()) - .map_err(|e| Error::ExecutionErr(e.to_string()))?; + let database = if let Some(db) = pg_args.get("database") { + serde_json::from_value::(db.clone()) + .map_err(|e| Error::ExecutionErr(e.to_string()))? + } else { + return Err(Error::BadRequest("Missing database argument".to_string())); + }; let sslmode = database.sslmode.unwrap_or("prefer".to_string()); let database_string = format!( "postgres://{user}:{password}@{host}:{port}/{dbname}?sslmode={sslmode}", @@ -107,13 +106,6 @@ pub async fn do_postgresql( (client, handle) }; - let args = &job - .args - .clone() - .unwrap_or_else(|| json!({})) - .as_object() - .map(|x| x.to_owned()) - .unwrap_or_else(|| json!({}).as_object().unwrap().to_owned()); let mut statement_values: Vec = vec![]; let sig = parse_pgsql_sig(&query) @@ -121,7 +113,12 @@ pub async fn do_postgresql( .args; for arg in &sig { - statement_values.push(args.get(&arg.name).unwrap_or(&json!(null)).clone()); + statement_values.push( + pg_args + .get(&arg.name) + .map(|x| x.to_owned()) + .unwrap_or_else(|| serde_json::Value::Null), + ); } let query_params = statement_values @@ -152,7 +149,7 @@ pub async fn do_postgresql( handle.abort(); // And then check that we got back the same string we sent over. - return Ok(result); + return Ok(to_raw_value(&result)); } #[derive(Debug)] diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 50efd3e990..99e8f7e2e5 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -2,6 +2,7 @@ use std::{collections::HashMap, process::Stdio}; use itertools::Itertools; use regex::Regex; +use serde_json::value::RawValue; use sqlx::{Pool, Postgres}; use tokio::{ fs::{metadata, DirBuilder, File}, @@ -64,6 +65,7 @@ pub async fn pip_compile( job_id: &Uuid, requirements: &str, logs: &mut String, + mem_peak: &mut i32, job_dir: &str, db: &Pool, worker_name: &str, @@ -128,6 +130,7 @@ pub async fn pip_compile( job_id, db, logs, + mem_peak, child_process, false, worker_name, @@ -164,13 +167,14 @@ pub async fn handle_python_job( worker_name: &str, job: &QueuedJob, logs: &mut String, + mem_peak: &mut i32, db: &sqlx::Pool, client: &AuthedClientBackgroundTask, inner_content: &String, shared_mount: &str, base_internal_url: &str, envs: HashMap, -) -> windmill_common::error::Result { +) -> windmill_common::error::Result> { create_dependencies_dir(job_dir).await; let mut additional_python_paths: Vec = WORKER_CONFIG @@ -199,6 +203,7 @@ pub async fn handle_python_job( &job.id, &requirements, logs, + mem_peak, job_dir, db, worker_name, @@ -221,6 +226,7 @@ pub async fn handle_python_job( &job.id, &job.workspace_id, logs, + mem_peak, db, worker_name, job_dir, @@ -284,7 +290,6 @@ pub async fn handle_python_job( }) .collect::>() .join(""); - let client = client.get_authed().await; create_args_and_out_file(&client, job, job_dir, db).await?; let import_loader = if relative_imports { @@ -385,6 +390,7 @@ except Exception as e: ); write_file(job_dir, "wrapper.py", &wrapper_content).await?; + let client = client.get_authed().await; let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?; let additional_python_paths_folders = additional_python_paths.iter().join(":"); if !*DISABLE_NSJAIL { @@ -471,6 +477,7 @@ mount {{ &job.id, db, logs, + mem_peak, child, !*DISABLE_NSJAIL, worker_name, @@ -488,6 +495,7 @@ pub async fn handle_python_reqs( job_id: &Uuid, w_id: &str, logs: &mut String, + mem_peak: &mut i32, db: &sqlx::Pool, worker_name: &str, job_dir: &str, @@ -637,6 +645,7 @@ pub async fn handle_python_reqs( &job_id, db, logs, + mem_peak, child, false, worker_name, diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 8fec698f73..5730065ca4 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -2,17 +2,17 @@ use base64::{engine, Engine as _}; use core::fmt::Write; use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; use pem; -use serde_json::{json, Value}; +use serde_json::{json, value::RawValue, Value}; use sha2::{Digest, Sha256}; -use windmill_common::error::Error; use windmill_common::jobs::QueuedJob; +use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::parse_snowflake_sig; use windmill_queue::HTTP_CLIENT; use serde::{Deserialize, Serialize}; -use crate::{common::transform_json_value, AuthedClient}; +use crate::{common::build_args_values, AuthedClientBackgroundTask}; #[derive(Serialize)] struct Claims { @@ -61,19 +61,12 @@ struct SnowflakeError { } pub async fn do_snowflake( - job: QueuedJob, - client: &AuthedClient, + job: &QueuedJob, + client: &AuthedClientBackgroundTask, query: &str, db: &sqlx::Pool, -) -> windmill_common::error::Result { - let args = if let Some(args) = &job.args { - Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?) - } else { - None - }; - - let snowflake_args: Value = serde_json::from_value(args.unwrap_or_else(|| json!({}))) - .map_err(|e| Error::ExecutionErr(e.to_string()))?; +) -> windmill_common::error::Result> { + let snowflake_args = build_args_values(job, client, db).await?; let database = serde_json::from_value::( snowflake_args.get("database").unwrap_or(&json!({})).clone(), @@ -107,14 +100,6 @@ pub async fn do_snowflake( let token = encode(&Header::new(Algorithm::RS256), &claims, &private_key) .map_err(|e| Error::ExecutionErr(e.to_string()))?; - let args = &job - .args - .clone() - .unwrap_or_else(|| json!({})) - .as_object() - .map(|x| x.to_owned()) - .unwrap_or_else(|| json!({}).as_object().unwrap().to_owned()); - let mut bindings = serde_json::Map::new(); let sig = parse_snowflake_sig(&query) .map_err(|x| Error::ExecutionErr(x.to_string()))? @@ -123,7 +108,7 @@ pub async fn do_snowflake( let mut i = 1; for arg in &sig { let arg_t = arg.otyp.clone().unwrap_or_else(|| "string".to_string()); - let arg_v = args.get(&arg.name).cloned().unwrap_or(json!("")); + let arg_v = snowflake_args.get(&arg.name).cloned().unwrap_or(json!("")); let snowflake_v = convert_typ_val(arg_t, arg_v); bindings.insert(i.to_string(), snowflake_v); @@ -187,20 +172,24 @@ pub async fn do_snowflake( )); } - let rows = result - .data - .iter() - .map(|row| { - let mut row_map = serde_json::Map::new(); - row.iter() - .zip(result.resultSetMetaData.rowType.iter()) - .for_each(|(val, row_type)| { - row_map - .insert(row_type.name.clone(), parse_val(&val, &row_type.r#type)); - }); - Value::from(row_map) - }) - .collect(); + let rows = to_raw_value( + &result + .data + .iter() + .map(|row| { + let mut row_map = serde_json::Map::new(); + row.iter() + .zip(result.resultSetMetaData.rowType.iter()) + .for_each(|(val, row_type)| { + row_map.insert( + row_type.name.clone(), + parse_val(&val, &row_type.r#type), + ); + }); + Value::from(row_map) + }) + .collect::>(), + ); Ok(rows) } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 1157bf8cdd..0d9ff22ca4 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -9,11 +9,10 @@ use anyhow::Result; use const_format::concatcp; use itertools::Itertools; -use once_cell::sync::OnceCell; use prometheus::core::{AtomicU64, GenericCounter}; -#[cfg(feature = "benchmark")] -use serde::Serialize; -use sqlx::{Pool, Postgres}; +use reqwest::Response; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use sqlx::{types::Json, Pool, Postgres}; use std::{ collections::HashMap, sync::{ @@ -22,7 +21,6 @@ use std::{ }, time::Duration, }; -use windmill_api_client::Client; use uuid::Uuid; use windmill_common::{ @@ -32,14 +30,15 @@ use windmill_common::{ scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::SUPERADMIN_SECRET_EMAIL, utils::{rd_string, StripPath}, - worker::{update_ping, CLOUD_HOSTED, WORKER_CONFIG}, + worker::{to_raw_value, to_raw_value_owned, update_ping, CLOUD_HOSTED, WORKER_CONFIG}, DB, IS_READY, METRICS_ENABLED, }; use windmill_queue::{ - canceled_job_to_result, get_queued_job, pull, push, PushIsolationLevel, HTTP_CLIENT, + canceled_job_to_result, empty_args, get_queued_job, pull, push, PushArgs, PushIsolationLevel, + WrappedError, HTTP_CLIENT, }; -use serde_json::{json, Value}; +use serde_json::{json, value::RawValue, Value}; use tokio::{ fs::{symlink, DirBuilder}, @@ -71,7 +70,7 @@ use windmill_queue::{add_completed_job, add_completed_job_error}; use crate::{ bash_executor::{handle_bash_job, handle_powershell_job, ANSI_ESCAPE_RE}, bun_executor::{gen_lockfile, get_trusted_deps, handle_bun_job}, - common::{hash_args, read_result, save_in_cache, transform_json_value, write_file}, + common::{build_args_map, hash_args, read_result, save_in_cache, write_file}, deno_executor::{generate_deno_lock, handle_deno_job}, go_executor::{handle_go_job, install_go_dependencies}, graphql_executor::do_graphql, @@ -98,16 +97,17 @@ pub async fn create_token_for_owner_in_bg( if job.workspace_id != "" { let mut locked = rw_lock.clone().write_owned().await; let db = db.clone(); - let job = job.clone(); + let w_id = job.workspace_id.clone(); + let owner = job.permissioned_as.clone(); + let email = job.email.clone(); tokio::spawn(async move { - let job = job.clone(); let token = create_token_for_owner( &db.clone(), - &job.workspace_id, - &job.permissioned_as, + &w_id, + &owner, "ephemeral-script", *SCRIPT_TOKEN_EXPIRY, - &job.email, + &email, ) .await .expect("could not create job token"); @@ -293,7 +293,6 @@ pub struct AuthedClientBackgroundTask { pub base_internal_url: String, pub workspace: String, pub token: Arc>, - pub client: OnceCell, } impl AuthedClientBackgroundTask { @@ -302,7 +301,6 @@ impl AuthedClientBackgroundTask { base_internal_url: self.base_internal_url.clone(), workspace: self.workspace.clone(), token: self.get_token().await, - client: self.client.clone(), }; } pub async fn get_token(&self) -> String { @@ -314,14 +312,110 @@ pub struct AuthedClient { pub base_internal_url: String, pub workspace: String, pub token: String, - pub client: OnceCell, } impl AuthedClient { - pub fn get_client(&self) -> &Client { - return self.client.get_or_init(|| { - windmill_api_client::create_client(&self.base_internal_url, self.token.clone()) - }); + pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result { + Ok(HTTP_CLIENT + .get(url) + .query(&query) + .header( + reqwest::header::ACCEPT, + reqwest::header::HeaderValue::from_static("application/json"), + ) + .header( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))?, + ) + .send() + .await?) + } + + pub async fn get_resource_value(&self, path: &str) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/resources/get_value/{}", + self.base_internal_url, self.workspace, path + ); + let response = self.get(&url, vec![]).await?; + match response.status().as_u16() { + 200u16 => Ok(response.json::().await?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } + + pub async fn get_variable_value(&self, path: &str) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/variables/get_value/{}", + self.base_internal_url, self.workspace, path + ); + let response = self.get(&url, vec![]).await?; + match response.status().as_u16() { + 200u16 => Ok(response.json::().await?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } + + pub async fn get_resource_value_interpolated( + &self, + path: &str, + job_id: Option, + ) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/resources/get_value_interpolated/{}", + self.base_internal_url, self.workspace, path + ); + let mut query = Vec::with_capacity(1usize); + if let Some(v) = &job_id { + query.push(("job_id", v.to_string())); + } + let response = self.get(&url, query).await?; + match response.status().as_u16() { + 200u16 => Ok(response.json::().await?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } + + pub async fn get_completed_job_result( + &self, + path: &str, + json_path: Option, + ) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/jobs_u/completed/get_result/{}", + self.base_internal_url, self.workspace, path + ); + let query = if let Some(json_path) = json_path { + vec![("json_path", json_path)] + } else { + vec![] + }; + let response = self.get(&url, query).await?; + match response.status().as_u16() { + 200u16 => Ok(response.json::().await?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } + + pub async fn get_result_by_id( + &self, + flow_job_id: &str, + node_id: &str, + json_path: Option, + ) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/jobs/result_by_id/{}/{}", + self.base_internal_url, self.workspace, flow_job_id, node_id + ); + let query = if let Some(json_path) = json_path { + vec![("json_path", json_path)] + } else { + vec![] + }; + let response = self.get(&url, query).await?; + match response.status().as_u16() { + 200u16 => Ok(response.json::().await?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } } } @@ -357,14 +451,12 @@ async fn handle_receive_completed_job< let metrics = build_language_metrics(&worker_execution_failed.clone(), &jc.job.language); let token = jc.token.clone(); let workspace = jc.job.workspace_id.clone(); - let client = AuthedClient { - base_internal_url: base_internal_url.to_string(), - workspace, - token, - client: OnceCell::new(), - }; + let client = + AuthedClient { base_internal_url: base_internal_url.to_string(), workspace, token }; + let job = jc.job.clone(); + let mem_peak = jc.mem_peak.clone(); if let Err(err) = process_completed_job( - &jc, + jc, &client, &db, &worker_dir, @@ -377,7 +469,8 @@ async fn handle_receive_completed_job< handle_job_error( &db, &client, - &jc.job, + job.as_ref(), + mem_peak, err, metrics, false, @@ -795,7 +888,7 @@ pub async fn run_worker(MAX_BUFFERED_DEDICATED_JOBS); + mpsc::channel::>(MAX_BUFFERED_DEDICATED_JOBS); let mut killpill_rx = killpill_rx.resubscribe(); let db = db.clone(); let worker_dir = worker_dir.clone(); @@ -891,7 +984,7 @@ pub async fn run_worker>, Option>) + (None, None) as (Option>>, Option>) }; #[cfg(feature = "benchmark")] @@ -1080,7 +1173,7 @@ pub async fn run_worker( concurrency_time_window_s: None, cache_ttl: None, }), - serde_json::Map::new(), + PushArgs::empty(), worker_name, "worker@windmill.dev", SUPERADMIN_SECRET_EMAIL.to_string(), @@ -1345,7 +1440,7 @@ async fn queue_init_bash_maybe<'c, R: rsmq_async::RsmqConnection + Send + 'c>( // ) -> error::Result<()> { pub async fn process_completed_job( - JobCompleted { job, result, logs, success, cached_res_path, .. }: &JobCompleted, + JobCompleted { job, result, logs, mem_peak, success, cached_res_path, .. }: JobCompleted, client: &AuthedClient, db: &DB, worker_dir: &str, @@ -1353,18 +1448,19 @@ pub async fn process_completed_job, rsmq: Option, ) -> windmill_common::error::Result<()> { - if *success { + if success { // println!("bef completed job{:?}", SystemTime::now()); if let Some(cached_path) = cached_res_path { - save_in_cache(&client, &job, cached_path.to_string(), &result).await; + save_in_cache(db, &job, cached_path.to_string(), &result).await; } add_completed_job( db, &job, true, false, - result, - logs.to_string(), + Json(&result), + logs, + mem_peak.to_owned(), rsmq.clone(), ) .await?; @@ -1377,7 +1473,7 @@ pub async fn process_completed_job, client: &AuthedClient, job: &QueuedJob, + mem_peak: i32, err: Error, metrics: Option, unrecoverable: bool, @@ -1479,6 +1577,7 @@ pub async fn handle_job_error serde_json::Value { - return json!({"message": format!("ExitCode: {i}, last log lines:\n{}", ANSI_ESCAPE_RE.replace_all(log_lines.trim(), "").to_string()), "name": "ExecutionErr"}); +fn extract_error_value(log_lines: &str, i: i32) -> Box { + return to_raw_value( + &json!({"message": format!("ExitCode: {i}, last log lines:\n{}", ANSI_ESCAPE_RE.replace_all(log_lines.trim(), "").to_string()), "name": "ExecutionErr"}), + ); } #[derive(Debug, Clone)] pub struct JobCompleted { - pub job: QueuedJob, - pub result: serde_json::Value, + pub job: Arc, + pub result: Box, pub logs: String, + pub mem_peak: i32, pub success: bool, pub cached_res_path: Option, pub token: String, @@ -1582,32 +1686,38 @@ pub async fn get_content(job: &QueuedJob, db: &Pool) -> Result, -) -> windmill_common::error::Result<(serde_json::Value, String)> { - let args = if let Some(args) = &job.args { - Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?) +) -> windmill_common::error::Result<(Box, String)> { + let args = build_args_map(job, client, db).await?.map(Json); + let job_args = if args.is_some() { + args.as_ref() } else { - None + job.args.as_ref() }; - let args = args - .as_ref() - .map(|x| x.clone()) - .unwrap_or_else(|| json!({})) - .as_object() - .unwrap() - .clone(); - let result = eval_fetch_timeout(code.clone(), transpile_ts(code)?, args).await?; + let result = eval_fetch_timeout(code.clone(), transpile_ts(code)?, job_args).await?; Ok((result.0, [logs, result.1].join("\n\n"))) } +#[derive(Deserialize)] +struct CachedResource { + expire: i64, + value: Box, +} + +#[derive(Deserialize, Serialize, Default)] +pub struct PreviousResult<'a> { + #[serde(borrow)] + pub previous_result: Option<&'a RawValue>, +} + #[tracing::instrument(level = "trace", skip_all)] async fn handle_queued_job( - job: QueuedJob, + job: Arc, db: &DB, client: &AuthedClientBackgroundTask, worker_name: &str, @@ -1619,10 +1729,10 @@ async fn handle_queued_job( job_completed_tx: Sender, ) -> windmill_common::error::Result<()> { if job.canceled { - return Err(Error::JsonErr(canceled_job_to_result(&job)))?; + return Err(Error::JsonErr(canceled_job_to_result(&job))); } - if let Some(e) = job.pre_run_error { - return Err(Error::ExecutionErr(e)); + if let Some(e) = &job.pre_run_error { + return Err(Error::ExecutionErr(e.to_string())); } let step = if job.is_flow_step { @@ -1641,7 +1751,7 @@ async fn handle_queued_job( }; let cached_res_path = if job.cache_ttl.is_some() { - let args_hash = hash_args(&job.args.clone().unwrap_or_else(|| json!({}))); + let args_hash = hash_args(&job.args); if job.is_flow_step { let flow_path = sqlx::query_scalar!( "SELECT script_path FROM queue WHERE id = $1", @@ -1665,54 +1775,42 @@ async fn handle_queued_job( if let Some(cached_res_path) = cached_res_path.clone() { let authed_client = client.get_authed().await; - let client: &Client = authed_client.get_client(); - let resource = client - .get_resource_value(&job.workspace_id, &cached_res_path) + let resource = authed_client + .get_resource_value::(&cached_res_path) .await; if let Ok(resource) = resource { - let v = resource.into_inner(); - if let Some(o) = v.as_object() { - let expire = o.get("expire"); - if expire.is_some() - && expire - .unwrap() - .as_i64() - .map(|x| x > chrono::Utc::now().timestamp()) - .unwrap_or(false) - { - let result = v - .get("value") - .map(|x| x.to_owned()) - .unwrap_or_else(|| json!({})); - let logs = "Job skipped because args & path found in cache and not expired" - .to_string(); + let expire = resource.expire; + if expire > chrono::Utc::now().timestamp() { + let result = resource.value; + let logs = + "Job skipped because args & path found in cache and not expired".to_string(); - job_completed_tx - .send(JobCompleted { - job, - result, - logs, - success: true, - cached_res_path: None, - token: authed_client.token, - }) - .await - .expect("send job completed"); + job_completed_tx + .send(JobCompleted { + job: job, + result, + logs, + mem_peak: 0, + success: true, + cached_res_path: None, + token: authed_client.token, + }) + .await + .expect("send job completed"); - return Ok(()); - } + return Ok(()); } } }; match job.job_kind { JobKind::FlowPreview | JobKind::Flow => { - let args = job.args.clone().unwrap_or(Value::Null); + let args = job.get_args(); handle_flow( &job, db, &client.get_authed().await, - args, + to_raw_value(&args), same_worker_tx, worker_dir, rsmq, @@ -1721,6 +1819,7 @@ async fn handle_queued_job( } _ => { let mut logs = "".to_string(); + let mut mem_peak: i32 = 0; // println!("handle queue {:?}", SystemTime::now()); if let Some(log_str) = &job.logs { logs.push_str(&log_str); @@ -1745,6 +1844,7 @@ async fn handle_queued_job( handle_dependency_job( &job, &mut logs, + &mut mem_peak, job_dir, db, worker_name, @@ -1757,6 +1857,7 @@ async fn handle_queued_job( JobKind::FlowDependencies => handle_flow_dependency_job( &job, &mut logs, + &mut mem_peak, job_dir, db, worker_name, @@ -1765,10 +1866,11 @@ async fn handle_queued_job( &client.get_token().await, ) .await - .map(|()| Value::Null), + .map(|()| serde_json::from_str("{}").unwrap()), JobKind::AppDependencies => handle_app_dependency_job( &job, &mut logs, + &mut mem_peak, job_dir, db, worker_name, @@ -1777,23 +1879,23 @@ async fn handle_queued_job( &client.get_token().await, ) .await - .map(|()| Value::Null), - JobKind::Identity => match job.args.clone() { - Some(Value::Object(args)) - if args.len() == 1 && args.contains_key("previous_result") => - { - Ok(args.get("previous_result").unwrap().clone()) - } - args @ _ => Ok(args.unwrap_or_else(|| Value::Null)), - }, + .map(|()| serde_json::from_str("{}").unwrap()), + JobKind::Identity => Ok(job + .args + .as_ref() + .map(|x| x.get("previous_result")) + .flatten() + .map(|x| x.to_owned()) + .unwrap_or_else(|| serde_json::from_str("{}").unwrap())), _ => { handle_code_execution_job( - &job, + job.as_ref(), db, client, job_dir, worker_dir, &mut logs, + &mut mem_peak, base_internal_url, worker_name, ) @@ -1802,7 +1904,7 @@ async fn handle_queued_job( }; //it's a test job, no need to update the db - if job.workspace_id == "" { + if job.as_ref().workspace_id == "" { return Ok(()); } process_result( @@ -1811,6 +1913,7 @@ async fn handle_queued_job( job_dir, job_completed_tx, logs, + mem_peak, cached_res_path, client.get_token().await, ) @@ -1821,11 +1924,12 @@ async fn handle_queued_job( } async fn process_result( - job: QueuedJob, - result: error::Result, + job: Arc, + result: error::Result>, job_dir: &str, job_completed_tx: Sender, logs: String, + mem_peak: i32, cached_res_path: Option, token: String, ) -> error::Result<()> { @@ -1833,9 +1937,10 @@ async fn process_result( Ok(r) => { job_completed_tx .send(JobCompleted { - job, + job: job, result: r, logs, + mem_peak, success: true, cached_res_path, token: token, @@ -1848,7 +1953,7 @@ async fn process_result( Error::ExitStatus(i) => { let res = read_result(job_dir).await.ok(); - if res.is_some() && res.clone().unwrap().is_object() { + if res.is_some() { res.unwrap() } else { let last_10_log_lines = logs @@ -1866,17 +1971,18 @@ async fn process_result( extract_error_value(log_lines, i) } } - err @ _ => { - json!({"message": format!("error during execution of the script:\n{}", err), "name": "ExecutionErr"}) - } + err @ _ => to_raw_value( + &json!({"message": format!("error during execution of the script:\n{}", err), "name": "ExecutionErr"}), + ), }; // in the happy path and if job not a flow step, we can delegate updating the completed job in the background job_completed_tx .send(JobCompleted { - job, - result: error_value, + job: job, + result: to_raw_value(&error_value), logs: logs, + mem_peak, success: false, cached_res_path, token: token, @@ -1927,9 +2033,10 @@ async fn handle_code_execution_job( job_dir: &str, worker_dir: &str, logs: &mut String, + mem_peak: &mut i32, base_internal_url: &str, worker_name: &str, -) -> error::Result { +) -> error::Result> { let (inner_content, requirements_o, language, envs) = match job.job_kind { JobKind::Preview => ( job.raw_code @@ -1975,9 +2082,9 @@ async fn handle_code_execution_job( }; if language == Some(ScriptLang::Postgresql) { - return do_postgresql(job.clone(), &client.get_authed().await, &inner_content, db).await; + return do_postgresql(job, &client, &inner_content, db).await; } else if language == Some(ScriptLang::Mysql) { - return do_mysql(job.clone(), &client.get_authed().await, &inner_content, db).await; + return do_mysql(job, &client, &inner_content, db).await; } else if language == Some(ScriptLang::Bigquery) { #[cfg(not(feature = "enterprise"))] { @@ -1988,7 +2095,7 @@ async fn handle_code_execution_job( #[cfg(feature = "enterprise")] { - return do_bigquery(job.clone(), &client.get_authed().await, &inner_content, db).await; + return do_bigquery(job, &client, &inner_content, db).await; } } else if language == Some(ScriptLang::Snowflake) { #[cfg(not(feature = "enterprise"))] @@ -2000,10 +2107,10 @@ async fn handle_code_execution_job( #[cfg(feature = "enterprise")] { - return do_snowflake(job.clone(), &client.get_authed().await, &inner_content, db).await; + return do_snowflake(job, &client, &inner_content, db).await; } } else if language == Some(ScriptLang::Graphql) { - return do_graphql(job.clone(), &client.get_authed().await, &inner_content, db).await; + return do_graphql(job, &client, &inner_content, db).await; } else if language == Some(ScriptLang::Nativets) { logs.push_str("\n--- FETCH TS EXECUTION ---\n"); let code = format!( @@ -2011,14 +2118,7 @@ async fn handle_code_execution_job( &client.get_token().await, inner_content ); - let (result, ts_logs) = do_nativets( - job.clone(), - logs.clone(), - &client.get_authed().await, - code, - db, - ) - .await?; + let (result, ts_logs) = do_nativets(job, logs.clone(), &client, code, db).await?; *logs = ts_logs; return Ok(result); } @@ -2057,7 +2157,7 @@ mount {{ let envs = build_envs(envs)?; - let result: error::Result = match language { + let result: error::Result> = match language { None => { return Err(Error::ExecutionErr( "Require language to be not null".to_string(), @@ -2071,6 +2171,7 @@ mount {{ worker_name, job, logs, + mem_peak, db, client, &inner_content, @@ -2084,6 +2185,7 @@ mount {{ handle_deno_job( requirements_o, logs, + mem_peak, job, db, client, @@ -2099,6 +2201,7 @@ mount {{ handle_bun_job( requirements_o, logs, + mem_peak, job, db, client, @@ -2114,6 +2217,7 @@ mount {{ Some(ScriptLang::Go) => { handle_go_job( logs, + mem_peak, job, db, client, @@ -2130,6 +2234,7 @@ mount {{ Some(ScriptLang::Bash) => { handle_bash_job( logs, + mem_peak, job, db, client, @@ -2145,6 +2250,7 @@ mount {{ Some(ScriptLang::Powershell) => { handle_powershell_job( logs, + mem_peak, job, db, client, @@ -2177,13 +2283,14 @@ mount {{ async fn handle_dependency_job( job: &QueuedJob, logs: &mut String, + mem_peak: &mut i32, job_dir: &str, db: &sqlx::Pool, worker_name: &str, worker_dir: &str, base_internal_url: &str, token: &str, -) -> error::Result { +) -> error::Result> { let content = capture_dependency_job( &job.id, job.language.as_ref().map(|v| Ok(v)).unwrap_or_else(|| { @@ -2196,6 +2303,7 @@ async fn handle_dependency_job( .map(|a| a.as_str()) .unwrap_or_else(|| "no raw code"), logs, + mem_peak, job_dir, db, worker_name, @@ -2216,7 +2324,9 @@ async fn handle_dependency_job( ) .execute(db) .await?; - Ok(json!({ "success": "Successful lock file generation", "lock": content })) + Ok(to_raw_value_owned( + json!({ "success": "Successful lock file generation", "lock": content }), + )) } Err(error) => { sqlx::query!( @@ -2235,6 +2345,7 @@ async fn handle_dependency_job( async fn handle_flow_dependency_job( job: &QueuedJob, logs: &mut String, + mem_peak: &mut i32, job_dir: &str, db: &sqlx::Pool, worker_name: &str, @@ -2258,6 +2369,7 @@ async fn handle_flow_dependency_job( flow.modules, job, logs, + mem_peak, job_dir, db, worker_name, @@ -2298,6 +2410,7 @@ async fn lock_modules( modules: Vec, job: &QueuedJob, logs: &mut String, + mem_peak: &mut i32, job_dir: &str, db: &sqlx::Pool, worker_name: &str, @@ -2333,6 +2446,7 @@ async fn lock_modules( modules, job, logs, + mem_peak, job_dir, db, worker_name, @@ -2354,6 +2468,7 @@ async fn lock_modules( b.modules, job, logs, + mem_peak, job_dir, db, worker_name, @@ -2374,6 +2489,7 @@ async fn lock_modules( b.modules, job, logs, + mem_peak, job_dir, db, worker_name, @@ -2389,6 +2505,7 @@ async fn lock_modules( default, job, logs, + mem_peak, job_dir, db, worker_name, @@ -2422,6 +2539,7 @@ async fn lock_modules( &language, &dependencies, logs, + mem_peak, job_dir, db, worker_name, @@ -2479,6 +2597,7 @@ async fn lock_modules_app( value: Value, job: &QueuedJob, logs: &mut String, + mem_peak: &mut i32, job_dir: &str, db: &sqlx::Pool, worker_name: &str, @@ -2521,6 +2640,7 @@ async fn lock_modules_app( &language, &dependencies, logs, + mem_peak, job_dir, db, worker_name, @@ -2560,6 +2680,7 @@ async fn lock_modules_app( b, job, logs, + mem_peak, job_dir, db, worker_name, @@ -2581,6 +2702,7 @@ async fn lock_modules_app( b, job, logs, + mem_peak, job_dir, db, worker_name, @@ -2601,6 +2723,7 @@ async fn lock_modules_app( async fn handle_app_dependency_job( job: &QueuedJob, logs: &mut String, + mem_peak: &mut i32, job_dir: &str, db: &sqlx::Pool, worker_name: &str, @@ -2628,6 +2751,7 @@ async fn handle_app_dependency_job( value, job, logs, + mem_peak, job_dir, db, worker_name, @@ -2665,6 +2789,7 @@ async fn capture_dependency_job( job_language: &ScriptLang, job_raw_code: &str, logs: &mut String, + mem_peak: &mut i32, job_dir: &str, db: &sqlx::Pool, worker_name: &str, @@ -2677,8 +2802,17 @@ async fn capture_dependency_job( match job_language { ScriptLang::Python3 => { create_dependencies_dir(job_dir).await; - let req: std::result::Result = - pip_compile(job_id, job_raw_code, logs, job_dir, db, worker_name, w_id).await; + let req: std::result::Result = pip_compile( + job_id, + job_raw_code, + logs, + mem_peak, + job_dir, + db, + worker_name, + w_id, + ) + .await; // install the dependencies to pre-fill the cache if let Ok(req) = req.as_ref() { let r = handle_python_reqs( @@ -2686,6 +2820,7 @@ async fn capture_dependency_job( job_id, w_id, logs, + mem_peak, db, worker_name, job_dir, @@ -2708,6 +2843,7 @@ async fn capture_dependency_job( job_id, job_raw_code, logs, + mem_peak, job_dir, db, false, @@ -2723,6 +2859,7 @@ async fn capture_dependency_job( job_id, job_raw_code, logs, + mem_peak, job_dir, db, w_id, @@ -2737,6 +2874,7 @@ async fn capture_dependency_job( let trusted_deps = get_trusted_deps(job_raw_code); let req = gen_lockfile( logs, + mem_peak, job_id, w_id, db, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index ab04fc90e8..18c9f2ad02 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -7,14 +7,19 @@ */ use std::collections::HashMap; +use std::sync::Arc; use std::time::Duration; use crate::common::{hash_args, save_in_cache}; use crate::js_eval::{eval_timeout, IdContext}; -use crate::{AuthedClient, KEEP_JOB_DIR}; +use crate::{AuthedClient, PreviousResult, KEEP_JOB_DIR}; use anyhow::Context; use async_recursion::async_recursion; -use serde_json::{json, Map, Value}; +use serde::Serialize; +use serde_json::value::RawValue; +use serde_json::{json, Value}; +use sqlx::types::Json; +use sqlx::FromRow; use tokio::sync::mpsc::Sender; use tracing::instrument; use uuid::Uuid; @@ -22,6 +27,7 @@ use windmill_common::flow_status::{FlowStatusModuleWParent, Iterator, JobResult} use windmill_common::jobs::{ script_hash_to_tag_and_limits, script_path_to_payload, JobPayload, Metrics, QueuedJob, RawCode, }; +use windmill_common::worker::to_raw_value; use windmill_common::{ error::{self, to_anyhow, Error}, flow_status::{ @@ -32,6 +38,7 @@ use windmill_common::{ }; use windmill_queue::{ add_completed_job, add_completed_job_error, handle_maybe_scheduled_job, PushIsolationLevel, + WrappedError, }; type DB = sqlx::Pool; @@ -40,6 +47,7 @@ use windmill_queue::{canceled_job_to_result, get_queued_job, push, QueueTransact // #[instrument(level = "trace", skip_all)] pub async fn update_flow_status_after_job_completion< + 'a, R: rsmq_async::RsmqConnection + Send + Sync + Clone, >( db: &DB, @@ -48,7 +56,7 @@ pub async fn update_flow_status_after_job_completion< job_id_for_status: &Uuid, w_id: &str, success: bool, - result: serde_json::Value, + result: &'a RawValue, metrics: Option, unrecoverable: bool, same_worker_tx: Sender, @@ -75,14 +83,14 @@ pub async fn update_flow_status_after_job_completion< ) .await?; while let Some(nrec) = rec { - rec = update_flow_status_after_job_completion_internal( + rec = match update_flow_status_after_job_completion_internal( db, client, nrec.flow, &nrec.job_id_for_status, w_id, nrec.success, - nrec.result, + nrec.result.as_ref(), metrics.clone(), false, same_worker_tx.clone(), @@ -91,7 +99,29 @@ pub async fn update_flow_status_after_job_completion< nrec.skip_error_handler, rsmq.clone(), ) - .await?; + .await + { + Ok(j) => j, + Err(e) => { + update_flow_status_after_job_completion_internal( + db, + client, + nrec.flow, + &nrec.job_id_for_status, + w_id, + false, + &to_raw_value(&Json(&WrappedError { error: e.to_string() })), + metrics.clone(), + true, + same_worker_tx.clone(), + worker_dir, + nrec.stop_early_override, + nrec.skip_error_handler, + rsmq.clone(), + ) + .await? + } + } } Ok(()) } @@ -99,12 +129,27 @@ pub struct RecUpdateFlowStatusAfterJobCompletion { flow: uuid::Uuid, job_id_for_status: Uuid, success: bool, - result: serde_json::Value, + result: Box, stop_early_override: Option, skip_error_handler: bool, } + +#[derive(FromRow)] +pub struct BranchResults<'a> { + pub result: &'a RawValue, + pub id: Uuid, +} + +#[derive(FromRow)] +pub struct SkipIfStopped { + pub skip_if_stopped: Option, + pub stop_early_expr: Option, + pub args: Option>>>, +} + // #[instrument(level = "trace", skip_all)] pub async fn update_flow_status_after_job_completion_internal< + 'a, R: rsmq_async::RsmqConnection + Send + Sync + Clone, >( db: &DB, @@ -113,7 +158,7 @@ pub async fn update_flow_status_after_job_completion_internal< job_id_for_status: &Uuid, w_id: &str, mut success: bool, - result: serde_json::Value, + result: &'a RawValue, metrics: Option, unrecoverable: bool, same_worker_tx: Sender, @@ -139,11 +184,11 @@ pub async fn update_flow_status_after_job_completion_internal< })? .ok_or_else(|| Error::InternalErr(format!("requiring a previous status")))?; - let old_status = serde_json::from_value::(old_status_json) - .ok() - .ok_or_else(|| { - Error::InternalErr(format!("requiring status to be parsabled as FlowStatus")) - })?; + let old_status = serde_json::from_value::(old_status_json).or_else(|e| { + Err(Error::InternalErr(format!( + "requiring status to be parsable as FlowStatus: {e:?}" + ))) + })?; let module_index = usize::try_from(old_status.step).ok(); @@ -186,25 +231,39 @@ pub async fn update_flow_status_after_job_completion_internal< } else if is_failure_step { (false, false) } else { - let r = sqlx::query!( + let row = sqlx::query( " SELECT raw_flow->'modules'->$1::int->'stop_after_if'->>'expr' as stop_early_expr, (raw_flow->'modules'->$1::int->'stop_after_if'->>'skip_if_stopped')::bool as skip_if_stopped, args FROM queue WHERE id = $2 - ", - old_status.step, + ").bind( + old_status.step) + .bind( flow ) .fetch_one(db) .await .map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e}")))?; + let r = SkipIfStopped::from_row(&row)?; let stop_early = success && if let Some(expr) = r.stop_early_expr.clone() { - compute_bool_from_expr(expr, &r.args, result.clone(), None, Some(client), None) - .await? + compute_bool_from_expr( + expr, + Arc::new( + r.args + .map(|x| x.0) + .unwrap_or_else(|| serde_json::from_str("{}").unwrap()) + .to_owned(), + ), + Arc::new(result.to_owned()), + None, + Some(client), + None, + ) + .await? } else { false }; @@ -446,20 +505,23 @@ pub async fn update_flow_status_after_job_completion_internal< let nresult = match &new_status { Some(FlowStatusModule::Success { flow_jobs: Some(jobs), .. }) | Some(FlowStatusModule::Failure { flow_jobs: Some(jobs), .. }) => { - let results = sqlx::query!( + let results = sqlx::query( " SELECT result, id FROM completed_job WHERE id = ANY($1) AND workspace_id = $2 ", - jobs.as_slice(), - w_id ) + .bind(jobs.as_slice()) + .bind(w_id) .fetch_all(&mut tx) .await? .into_iter() - .map(|r| (r.id, r.result)) + .map(|r| { + let br = BranchResults::from_row(&r).unwrap(); + (br.id, br.result.to_owned()) + }) .collect::>(); let results = jobs @@ -471,9 +533,9 @@ pub async fn update_flow_status_after_job_completion_internal< }) .collect::, _>>()?; - json!(results) + to_raw_value(&results) } - _ => result, + _ => result.to_owned(), }; if matches!(&new_status, Some(FlowStatusModule::Success { .. })) { @@ -572,6 +634,7 @@ pub async fn update_flow_status_after_job_completion_internal< db, &flow_job, logs, + 0, &canceled_job_to_result(&flow_job), metrics.clone(), rsmq.clone(), @@ -580,20 +643,21 @@ pub async fn update_flow_status_after_job_completion_internal< } else { if flow_job.cache_ttl.is_some() { let cached_res_path = { - let args_hash = hash_args(&flow_job.args.clone().unwrap_or_else(|| json!({}))); + let args_hash = hash_args(&flow_job.args); let flow_path = flow_job.script_path(); format!("{flow_path}/flow/cache/{args_hash}") }; - save_in_cache(&client, &flow_job, cached_res_path, &nresult).await; + save_in_cache(db, &flow_job, cached_res_path, &nresult).await; } add_completed_job( db, &flow_job, success && !is_failure_step && !skip_error_handler, stop_early && skip_if_stop_early, - &nresult, + Json(&nresult), logs, + 0, rsmq.clone(), ) .await?; @@ -604,7 +668,7 @@ pub async fn update_flow_status_after_job_completion_internal< &flow_job, db, client, - nresult.clone(), + nresult.to_owned(), same_worker_tx.clone(), worker_dir, rsmq.clone(), @@ -617,6 +681,7 @@ pub async fn update_flow_status_after_job_completion_internal< db, &flow_job, "Unexpected error during flow chaining:\n".to_string(), + 0, &e, metrics.clone(), rsmq.clone(), @@ -739,31 +804,34 @@ fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u16, Duration)> { async fn compute_bool_from_expr( expr: String, - flow_args: &Option, - result: serde_json::Value, + flow_args: Arc>>, + result: Arc>, by_id: Option, client: Option<&AuthedClient>, - resumes: Option<(&[Value], Vec)>, + resumes: Option<(Arc>, Arc>, Arc>)>, ) -> error::Result { - let flow_input = flow_args.clone().unwrap_or_else(|| json!({})); - let mut env = vec![ - ("flow_input".to_string(), flow_input), - ("result".to_string(), result.clone()), - ("previous_result".to_string(), result), - ]; + let mut context = HashMap::with_capacity(if resumes.is_some() { 7 } else { 3 }); + context.insert("result".to_string(), result.clone()); + context.insert("previous_result".to_string(), result); if let Some(resumes) = resumes { - env.push(( - "resume".to_string(), - resumes.0.last().map(|v| json!(v)).unwrap_or_default(), - )); - env.push(("resumes".to_string(), resumes.0.into())); - env.push(("approvers".to_string(), json!(resumes.1.clone()))); + context.insert("resume".to_string(), resumes.1); + context.insert("resumes".to_string(), resumes.0); + context.insert("approvers".to_string(), resumes.2); } - match eval_timeout(expr, env.into(), client, by_id).await? { - serde_json::Value::Bool(true) => Ok(true), - serde_json::Value::Bool(false) => Ok(false), + match eval_timeout( + format!("Boolean({expr})"), + context, + Some(flow_args), + client, + by_id, + ) + .await? + .get() + { + "true" => Ok(true), + "false" => Ok(false), a @ _ => Err(Error::ExecutionErr(format!( "Expected a boolean value, found: {a:?}" ))), @@ -828,63 +896,59 @@ pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result { } } +#[derive(serde::Deserialize)] +pub struct ErrorValue<'a> { + #[serde(borrow)] + pub error: Option<&'a RawValue>, +} + /// resumes should be in order of timestamp ascending, so that more recent are at the end #[instrument(level = "trace", skip_all)] async fn transform_input( - flow_args: &Option, - last_result: serde_json::Value, + flow_args: Arc>>, + last_result: Arc>, input_transforms: &HashMap, - resumes: &[Value], - approvers: Vec, + resumes: Arc>, + resume: Arc>, + approvers: Arc>, by_id: &IdContext, client: &AuthedClient, -) -> windmill_common::error::Result> { - let mut mapped = serde_json::Map::new(); +) -> windmill_common::error::Result>> { + let mut mapped = HashMap::new(); - for (key, val) in input_transforms.into_iter() { - if let InputTransform::Static { value } = val { - mapped.insert(key.to_string(), value.to_owned()); - } + let mut env = HashMap::new(); + + if input_transforms + .iter() + .any(|x| matches!(x.1, InputTransform::Javascript { .. })) + { + env.insert("params".to_string(), Arc::new(to_raw_value(&mapped))); + env.insert("previous_result".to_string(), last_result); + env.insert("resume".to_string(), resume); + env.insert("resumes".to_string(), resumes); + env.insert("approvers".to_string(), approvers); } - let lresult = last_result.clone(); - let error = if let Some(error) = lresult.as_object() { - error.get("error") - } else { - None - }; - for (key, val) in input_transforms.into_iter() { match val { - InputTransform::Static { value: _ } => (), + InputTransform::Static { value } => { + mapped.insert(key.to_string(), to_raw_value(&value)); + } InputTransform::Javascript { expr } => { - let flow_input = flow_args.clone().unwrap_or_else(|| json!({})); - let previous_result = last_result.clone(); - let mut context = vec![ - ("params".to_string(), json!(mapped)), - ("previous_result".to_string(), previous_result), - ("flow_input".to_string(), flow_input), - ( - "resume".to_string(), - resumes.last().map(|v| json!(v)).unwrap_or_default(), - ), - ("resumes".to_string(), resumes.into()), - ("approvers".to_string(), json!(approvers.clone())), - ]; - - if error.is_some() { - context.push(("error".to_string(), error.unwrap().clone())); - } - - let v = eval_timeout(expr.to_string(), context, Some(client), Some(by_id.clone())) - .await - .map_err(|e| { - Error::ExecutionErr(format!( - "Error during isolated evaluation of expression `{expr}`:\n{e}" - )) - })?; + let v = eval_timeout( + expr.to_string(), + env.clone(), + Some(flow_args.clone()), + Some(client), + Some(by_id.clone()), + ) + .await + .map_err(|e| { + Error::ExecutionErr(format!( + "Error during isolated evaluation of expression `{expr}`:\n{e}" + )) + })?; mapped.insert(key.to_string(), v); - () } } } @@ -897,7 +961,7 @@ pub async fn handle_flow( flow_job: &QueuedJob, db: &sqlx::Pool, client: &AuthedClient, - last_result: serde_json::Value, + last_result: Box, same_worker_tx: Sender, worker_dir: &str, rsmq: Option, @@ -920,7 +984,7 @@ pub async fn handle_flow( flow, db, client, - last_result, + last_result.to_owned(), same_worker_tx, worker_dir, rsmq, @@ -929,6 +993,33 @@ pub async fn handle_flow( Ok(()) } +#[derive(Serialize, Debug)] +pub struct Iter { + index: i32, + value: serde_json::Value, +} + +#[derive(Serialize)] +pub struct MergeArgs<'a> { + #[serde(flatten)] + b: &'a Iter, + + #[serde(flatten)] + a: HashMap>, +} + +#[derive(FromRow)] +pub struct ResumeRow { + pub value: Json>, + pub approver: Option, + pub resume_id: i32, +} + +#[derive(FromRow)] +pub struct RawArgs { + pub args: Option>>>, +} + #[async_recursion] // #[instrument(level = "trace", skip_all)] async fn push_next_flow_job( @@ -937,7 +1028,7 @@ async fn push_next_flow_job flow: FlowValue, db: &sqlx::Pool, client: &AuthedClient, - mut last_result: serde_json::Value, + last_result: Box, same_worker_tx: Sender, worker_dir: &str, rsmq: Option, @@ -957,8 +1048,11 @@ async fn push_next_flow_job .cloned() .unwrap_or_else(|| status.failure_module.module_status.clone()); + let flow_job_args = flow_job.get_args(); + // if this is an empty module of if the module has aleady been completed, successfully, update the parent flow if flow.modules.is_empty() || matches!(status_module, FlowStatusModule::Success { .. }) { + let r; return update_flow_status_after_job_completion( db, client, @@ -967,9 +1061,11 @@ async fn push_next_flow_job flow_job.workspace_id.as_str(), true, if flow.modules.is_empty() { - flow_job.args.clone().unwrap_or_default() + r = to_raw_value(&flow_job_args); + &r } else { - json!([]) + // it has to be an empty for loop event + serde_json::from_str("[]").unwrap() }, None, true, @@ -981,94 +1077,42 @@ async fn push_next_flow_job .await; } - if let Some(skip_expr) = &flow.skip_expr { - let skip = compute_bool_from_expr( - skip_expr.to_string(), - &flow_job.args, - last_result.clone(), - None, - Some(client), - None, - ) - .await?; - if skip { - return update_flow_status_after_job_completion( - db, - client, - flow_job.id, - &Uuid::nil(), - flow_job.workspace_id.as_str(), - true, - json!([]), + let arc_flow_job_args = Arc::new(flow_job_args.clone()); + let mut arc_result = Arc::new(last_result); + + if i == 0 { + if let Some(skip_expr) = &flow.skip_expr { + let skip = compute_bool_from_expr( + skip_expr.to_string(), + arc_flow_job_args.clone(), + arc_result.clone(), + None, + Some(client), None, - true, - same_worker_tx, - worker_dir, - Some(true), - rsmq, ) - .await; + .await?; + if skip { + return update_flow_status_after_job_completion( + db, + client, + flow_job.id, + &Uuid::nil(), + flow_job.workspace_id.as_str(), + true, + serde_json::from_str("\"stopped early\"").unwrap(), + None, + true, + same_worker_tx, + worker_dir, + Some(true), + rsmq, + ) + .await; + } } } - let mut module: &FlowModule = flow - .modules - .get(i) - .or_else(|| flow.failure_module.as_ref()) - .with_context(|| format!("no module at index {}", status.step))?; - - let current_id = &module.id; - let previous_id = if i >= 1 { - flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap() - } else { - String::new() - }; - - // calculate sleep if any - let mut scheduled_for_o = { - let sleep_input_transform = i - .checked_sub(1) - .and_then(|i| flow.modules.get(i)) - .and_then(|m| m.sleep.clone()); - - if let Some(it) = sleep_input_transform { - let json_value = match it { - InputTransform::Static { value } => value, - InputTransform::Javascript { expr } => { - let flow_input = flow_job.args.clone().unwrap_or_else(|| json!({})); - - eval_timeout( - expr.to_string(), - [ - ("result".to_string(), last_result.clone()), - ("flow_input".to_string(), flow_input), - ] - .into(), - None, - None, - ) - .await - .map_err(|e| { - Error::ExecutionErr(format!( - "Error during isolated evaluation of expression `{expr}`:\n{e}" - )) - })? - } - }; - match json_value { - serde_json::Value::Number(n) => { - n.as_u64().map(|x| from_now(Duration::from_secs(x))) - } - _ => Err(Error::ExecutionErr(format!( - "Expected a number value, found: {json_value}" - )))?, - } - } else { - None - } - }; - - let mut resume_messages: Vec = vec![]; + let mut resume_messages: Vec> = vec![]; let mut approvers: Vec = vec![]; /* (suspend / resume), when starting a module, if previous module has a @@ -1094,16 +1138,25 @@ async fn push_next_flow_job .await .context("lock flow in queue")?; - let resumes = sqlx::query!( + let resumes = sqlx::query( "SELECT value, approver, resume_id FROM resume_job WHERE job = $1 ORDER BY created_at ASC", - last ) + .bind(last) .fetch_all(&mut *tx) - .await?; + .await? + .into_iter() + .map(|x| ResumeRow::from_row(&x)) + .collect::>(); - resume_messages.extend(resumes.iter().map(|r| r.value.clone())); + resume_messages.extend( + resumes + .iter() + .map(|r| to_raw_value(&r.as_ref().map(|x| x.value.clone()).ok())), + ); approvers.extend(resumes.iter().map(|r| { - r.approver + r.as_ref() + .ok() + .and_then(|x| x.approver.clone()) .as_deref() .unwrap_or_else(|| "anonymous") .to_string() @@ -1123,8 +1176,12 @@ async fn push_next_flow_job .bind(json!(resumes .into_iter() .map(|r| Approval { - resume_id: r.resume_id as u16, - approver: r.approver.unwrap_or_else(|| "unknown".to_string()) + resume_id: r.as_ref().map(|x| x.resume_id).unwrap_or_default() as u16, + approver: r + .as_ref() + .ok() + .and_then(|x| x.approver.clone()) + .unwrap_or_else(|| "unknown".to_string()) }) .collect::>())) .bind(flow_job.id) @@ -1134,11 +1191,15 @@ async fn push_next_flow_job /* If we are woken up after suspending, last_result will be the flow args, but we * should use the result from the last job */ if let FlowStatusModule::WaitingForEvents { .. } = &status_module { - last_result = - sqlx::query_scalar!("SELECT result FROM completed_job WHERE id = $1", last) - .fetch_one(&mut *tx) - .await? - .context("previous job result")?; + arc_result = Arc::new( + sqlx::query_scalar::<_, Json>>( + "SELECT result FROM completed_job WHERE id = $1", + ) + .bind(last) + .fetch_one(&mut *tx) + .await? + .0, + ) } /* continue on and run this job! */ @@ -1176,15 +1237,92 @@ async fn push_next_flow_job let skipped = false; let logs = "Timed out waiting to be resumed".to_string(); let result = json!({ "error": {"message": logs, "name": "SuspendedTimeout"}}); - let _uuid = - add_completed_job(db, &flow_job, success, skipped, &result, logs, rsmq).await?; + let _uuid = add_completed_job( + db, + &flow_job, + success, + skipped, + Json(&result), + logs, + 0, + rsmq, + ) + .await?; return Ok(()); } } } - match &status_module { + let mut module: &FlowModule = flow + .modules + .get(i) + .or_else(|| flow.failure_module.as_ref()) + .with_context(|| format!("no module at index {}", status.step))?; + + let current_id = &module.id; + let previous_id = if i >= 1 { + flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap() + } else { + String::new() + }; + + // calculate sleep if any + let mut scheduled_for_o = { + // avoid branchall sleeping on every iteration if sleep is on module prior + if !matches!( + &status_module, + FlowStatusModule::WaitingForPriorSteps { .. } + | FlowStatusModule::WaitingForEvents { .. } + ) { + None + } else { + let sleep_input_transform = i + .checked_sub(1) + .and_then(|i| flow.modules.get(i)) + .and_then(|m| m.sleep.clone()); + + if let Some(it) = sleep_input_transform { + let json_value = match it { + InputTransform::Static { value } => Ok(value), + InputTransform::Javascript { expr } => { + let mut context = HashMap::with_capacity(2); + context.insert("result".to_string(), arc_result.clone()); + context.insert("previous_result".to_string(), arc_result.clone()); + + serde_json::from_str( + eval_timeout( + expr.to_string(), + context, + Some(arc_flow_job_args.clone()), + None, + None, + ) + .await + .map_err(|e| { + Error::ExecutionErr(format!( + "Error during isolated evaluation of expression `{expr}`:\n{e}" + )) + })? + .get(), + ) + } + }; + match json_value { + Ok(serde_json::Value::Number(n)) => { + n.as_u64().map(|x| from_now(Duration::from_secs(x))) + } + _ => Err(Error::ExecutionErr(format!( + "Expected a number value, found: {json_value:?}" + )))?, + } + } else { + None + } + } + }; + + let get_args_from_id = match &status_module { FlowStatusModule::Failure { job, .. } => { let retry = &module.retry.clone().unwrap_or_default(); if let Some((fail_count, retry_in)) = next_retry(retry, &status.retry) { @@ -1209,14 +1347,9 @@ async fn push_next_flow_job .await .context("update flow retry")?; - /* it might be better to retry the job using the previous args instead of determining - * them again from the last result, but that seemed to not play well with the forloop - * logic and I couldn't figure out why. */ - if let Some(v) = &status.retry.previous_result { - last_result = v.clone(); - } status_module = FlowStatusModule::WaitingForPriorSteps { id: status_module.id() }; - + // we get the args from the last failed job + status.retry.failed_jobs.last() /* Start the failure module ... */ } else { /* push_next_flow_job is called with the current step on FlowStatusModule::Failure. @@ -1246,16 +1379,13 @@ async fn push_next_flow_job WHERE id = $2 ", ) - .bind(json!(RetryStatus { - previous_result: Some(last_result.clone()), - fail_count: 0, - failed_jobs: vec![], - })) + .bind(json!(RetryStatus { fail_count: 0, failed_jobs: vec![] })) .bind(flow_job.id) .execute(db) .await .context("update flow retry")?; }; + None } /* (retry feature) save the previous_result the first time this step is run */ @@ -1275,39 +1405,52 @@ async fn push_next_flow_job WHERE id = $2 ", ) - .bind(json!(RetryStatus { - previous_result: Some(last_result.clone()), - fail_count: 0, - failed_jobs: vec![], - })) + .bind(json!(RetryStatus { fail_count: 0, failed_jobs: vec![] })) .bind(flow_job.id) .execute(db) .await .context("update flow retry")?; + None } - _ => (), - } + _ => None, + }; let mut transform_context: Option = None; + let approvers = Arc::new(to_raw_value(&approvers)); + let resume = Arc::new(to_raw_value(&resume_messages.last())); + let resumes = Arc::new(to_raw_value(&resume_messages)); + + drop(resume_messages); + let args: windmill_common::error::Result<_> = if module.mock.is_some() && module.mock.as_ref().unwrap().enabled { - let mut m = Map::new(); - let v = module - .mock - .as_ref() - .unwrap() - .return_value - .clone() - .ok_or_else(|| { - Error::BadRequest(format!( - "mock enabled but no return_value specified for module {}", - module.id - )) - })?; - m.insert("previous_result".to_string(), v); - Ok(m) + let mut hm = HashMap::new(); + hm.insert( + "previous_result".to_string(), + module + .mock + .as_ref() + .unwrap() + .return_value + .clone() + .unwrap_or_else(|| serde_json::from_str("null").unwrap()), + ); + Ok(hm) + } else if let Some(id) = get_args_from_id { + let row = sqlx::query("SELECT args FROM completed_job WHERE id = $1 AND workspace_id = $2") + .bind(id) + .bind(&flow_job.workspace_id) + .fetch_optional(db) + .await?; + if let Some(row) = row { + RawArgs::from_row(&row) + .map(|x| x.args.map(|x| x.0).unwrap_or_else(HashMap::new)) + .map_err(|e| error::Error::InternalErr(format!("Impossible to build args: {e}"))) + } else { + Ok(HashMap::new()) + } } else { match &module.value { FlowModuleValue::Script { input_transforms, .. } @@ -1317,42 +1460,30 @@ async fn push_next_flow_job transform_context = Some(ctx); let by_id = transform_context.as_ref().unwrap(); transform_input( - &flow_job.args, - last_result.clone(), + arc_flow_job_args.clone(), + arc_result.clone(), input_transforms, - resume_messages.as_slice(), + resumes.clone(), + resume.clone(), approvers.clone(), by_id, client, ) .await } - FlowModuleValue::Identity => match last_result.clone() { - Value::Object(m) => Ok(m), - v @ _ => { - let mut m = Map::new(); - m.insert("previous_result".to_string(), v); - Ok(m) - } - }, + FlowModuleValue::Identity => serde_json::from_str( + &serde_json::to_string(&PreviousResult { previous_result: Some(&arc_result) }) + .unwrap(), + ) + .map_err(|e| error::Error::InternalErr(format!("identity: {e}"))), - _ => { - /* embedded flow input is augmented with embedding flow input */ - if let Some(value) = &flow_job.args { - Ok(value - .as_object() - .ok_or_else(|| { - Error::BadRequest(format!("Expected an object value, found: {value:?}")) - })? - .clone()) - } else { - Ok(Map::new()) - } - } + _ => Ok(flow_job_args), } }; let next_flow_transform = compute_next_flow_transform( + arc_flow_job_args.clone(), + arc_result.clone(), flow_job, &flow, transform_context, @@ -1360,10 +1491,10 @@ async fn push_next_flow_job &module, &status, &status_module, - last_result.clone(), &previous_id, client, - resume_messages.as_slice(), + resumes.clone(), + resume.clone(), approvers.clone(), ) .await?; @@ -1423,11 +1554,21 @@ async fn push_next_flow_job .. } => args.as_ref().map(|args| args.clone()), NextStatus::NextLoopIteration(NextIteration { new_args, .. }) => { - args.as_ref().map(|args| { - let mut args = args.clone(); - args.extend(new_args.clone()); - args - }) + let mut args = if let Ok(args) = args.as_ref() { + args.clone() + } else { + HashMap::new() + }; + args.insert("iter".to_string(), to_raw_value(new_args)); + Ok(args) + // tracing::error!( + // "{a:?} {new_args:?} {:?}", + // to_raw_value(&MergeArgs { a: HashMap::new(), b: new_args.to_owned() }) + // ); + // Ok(to_raw_value(&MergeArgs { + // a: HashMap::new(), + // b: new_args.to_owned(), + // })) } NextStatus::AllFlowJobs { branchall: None, @@ -1435,18 +1576,22 @@ async fn push_next_flow_job simple_input_transforms, } => { if let Ok(args) = args.as_ref() { - let mut new_args = args.clone(); - new_args.insert( + let mut hm = HashMap::new(); + hm.insert( "iter".to_string(), - json!({ "index": i, "value": itered[i] }), + to_raw_value(&json!({ "index": i as i32, "value": itered[i]})), ); + for (k, v) in args { + hm.insert(k.to_string(), v.to_owned()); + } if let Some(input_transforms) = simple_input_transforms { let ctx = get_transform_context(&flow_job, &previous_id, &status).await?; transform_inp = transform_input( - &Some(serde_json::Value::Object(new_args)), - last_result.clone(), + Arc::new(hm), + arc_result.clone(), input_transforms, - resume_messages.as_slice(), + resumes.clone(), + resume.clone(), approvers.clone(), &ctx, client, @@ -1454,7 +1599,7 @@ async fn push_next_flow_job .await; transform_inp.as_ref().map(|args| args.clone()) } else { - Ok(new_args) + Ok(hm) } } else { args.as_ref().map(|args| args.clone()) @@ -1481,7 +1626,7 @@ async fn push_next_flow_job tx2, &flow_job.workspace_id, payload_tag.payload, - ok.unwrap_or_else(|| Map::new()), + ok.unwrap_or_else(|| serde_json::from_str("{}").unwrap()), &flow_job.created_by, &flow_job.email, flow_job.permissioned_as.to_owned(), @@ -1706,13 +1851,13 @@ async fn push_next_flow_job #[derive(Debug)] struct NextIteration { index: usize, - itered: Vec, + itered: Vec, flow_jobs: Vec, - new_args: Map, + new_args: Iter, } enum LoopStatus { - ParallelIteration { itered: Vec }, + ParallelIteration { itered: Vec }, NextIteration(NextIteration), EmptyIterator, } @@ -1753,6 +1898,8 @@ enum NextFlowTransform { } async fn compute_next_flow_transform( + arc_flow_job_args: Arc>>, + arc_result: Arc>, flow_job: &QueuedJob, flow: &FlowValue, by_id: Option, @@ -1760,11 +1907,11 @@ async fn compute_next_flow_transform( module: &FlowModule, status: &FlowStatus, status_module: &FlowStatusModule, - last_result: serde_json::Value, previous_id: &str, client: &AuthedClient, - resumes: &[Value], - approvers: Vec, + resumes: Arc>, + resume: Arc>, + approvers: Arc>, ) -> error::Result { if module.mock.is_some() && module.mock.as_ref().unwrap().enabled { return Ok(NextFlowTransform::Continue( @@ -1827,7 +1974,6 @@ async fn compute_next_flow_transform( } /* forloop modules are expected set `iter: { value: Value, index: usize }` as job arguments */ FlowModuleValue::ForloopFlow { modules, iterator, parallel, .. } => { - let new_args: &mut Map = &mut Map::new(); // if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg let is_simple = modules.len() == 1 && modules[0].value.is_simple() && flow.failure_module.is_none(); @@ -1841,29 +1987,29 @@ async fn compute_next_flow_transform( } else { get_transform_context(&flow_job, previous_id, &status).await? }; - let flow_input = flow_job.args.clone().unwrap_or_else(|| json!({})); /* Iterator is an InputTransform, evaluate it into an array. */ - let itered = evaluate_with( - iterator.clone(), - || { - vec![ - ("flow_input".to_string(), flow_input), - ("result".to_string(), last_result.clone()), - ("previous_result".to_string(), last_result.clone()), - ( - "resume".to_string(), - resumes.last().map(|v| json!(v)).unwrap_or_default(), - ), - ("resumes".to_string(), resumes.into()), - ("approvers".to_string(), json!(approvers.clone())), - ] - }, - Some(client), - Some(by_id), - ) - .await? - .into_array() - .map_err(|not_array| { + let itered_raw = match iterator { + InputTransform::Static { value } => to_raw_value(value), + InputTransform::Javascript { expr } => { + let mut context = HashMap::with_capacity(3); + context.insert("result".to_string(), arc_result.clone()); + context.insert("previous_result".to_string(), arc_result); + context.insert("resumes".to_string(), resumes); + context.insert("resume".to_string(), resume); + context.insert("approvers".to_string(), approvers); + + eval_timeout( + expr.to_string(), + context, + Some(arc_flow_job_args), + Some(client), + Some(by_id), + ) + .await? + } + }; + let itered = serde_json::from_str::>(itered_raw.get()) + .map_err(|not_array| { Error::ExecutionErr(format!("Expected an array value, found: {not_array}")) })?; @@ -1872,13 +2018,12 @@ async fn compute_next_flow_transform( } else if *parallel { LoopStatus::ParallelIteration { itered } } else if let Some(first) = itered.first() { - new_args.insert("iter".to_string(), json!({ "index": 0, "value": first })); - + let iter = Iter { index: 0 as i32, value: first.to_owned() }; LoopStatus::NextIteration(NextIteration { index: 0, itered, flow_jobs: vec![], - new_args: new_args.clone(), + new_args: iter, }) } else { panic!("itered cannot be empty") @@ -1901,13 +2046,11 @@ async fn compute_next_flow_transform( format!("could not iterate index {index} of {itered:?}") })?; - new_args.insert("iter".to_string(), json!({ "index": index, "value": next })); - LoopStatus::NextIteration(NextIteration { index, itered: itered.clone(), flow_jobs: flow_jobs.clone(), - new_args: new_args.clone(), + new_args: Iter { index: index as i32, value: next.to_owned() }, }) } @@ -2036,11 +2179,11 @@ async fn compute_next_flow_transform( for (i, b) in branches.iter().enumerate() { let pred = compute_bool_from_expr( b.expr.to_string(), - &flow_job.args, - last_result.clone(), + arc_flow_job_args.clone(), + arc_result.clone(), Some(idcontext.clone()), Some(client), - Some((resumes, approvers.clone())), + Some((resumes.clone(), resume.clone(), approvers.clone())), ) .await?; @@ -2141,36 +2284,21 @@ async fn compute_next_flow_transform( .collect(), ), NextStatus::AllFlowJobs { - branchall: Some(BranchAllStatus { - branch: 0, - previous_result: last_result, - len: branches.len(), - }), + branchall: Some(BranchAllStatus { branch: 0, len: branches.len() }), iterator: None, simple_input_transforms: None, }, )); } else { - ( - BranchAllStatus { - branch: 0, - previous_result: last_result, - len: branches.len(), - }, - vec![], - ) + (BranchAllStatus { branch: 0, len: branches.len() }, vec![]) } } FlowStatusModule::InProgress { - branchall: Some(BranchAllStatus { branch, previous_result, len }), + branchall: Some(BranchAllStatus { branch, len }), flow_jobs: Some(flow_jobs), .. } if !*parallel => ( - BranchAllStatus { - branch: branch + 1, - previous_result: previous_result.clone(), - len: len.clone(), - }, + BranchAllStatus { branch: branch + 1, len: len.clone() }, flow_jobs.clone(), ), @@ -2298,20 +2426,6 @@ async fn get_transform_context( Ok(IdContext { flow_job: flow_job.id, steps_results, previous_id: previous_id.to_string() }) } -async fn evaluate_with( - transform: InputTransform, - vars: F, - client: Option<&AuthedClient>, - by_id: Option, -) -> anyhow::Result -where - F: FnOnce() -> Vec<(String, serde_json::Value)>, -{ - match transform { - InputTransform::Static { value } => Ok(value), - InputTransform::Javascript { expr } => eval_timeout(expr, vars(), client, by_id).await, - } -} trait IntoArray: Sized { fn into_array(self) -> Result, Self>; } diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 7bdb1538c5..e055011228 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -478,7 +478,7 @@ on:click={() => editor?.reloadWebsocket()} startIcon={{ icon: faRotate, - classes: !websocketAlive[lang] ? 'animate-spin' : '' + classes: websocketAlive[lang] == false ? 'animate-spin' : '' }} title="Reload assistants" > diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 709ce62b78..eae91a8d07 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -403,9 +403,12 @@

SSO

- - Without EE, the number of SSO users is limited to 50. SCIM/SAML is available on EE - + {#if !$enterpriseLicense} + + Without EE, the number of SSO users is limited to 50. SCIM/SAML is available on EE + + {/if} +
The recommended workflow is to to save your oauth setting and test them directly on the diff --git a/frontend/src/lib/components/ScheduleEditor.svelte b/frontend/src/lib/components/ScheduleEditor.svelte index c204179835..6e50584c65 100644 --- a/frontend/src/lib/components/ScheduleEditor.svelte +++ b/frontend/src/lib/components/ScheduleEditor.svelte @@ -1,451 +1,23 @@ - - - - {#if edit} -
- -
-
- { - await ScheduleService.setScheduleEnabled({ - path: initialPath, - workspace: $workspaceStore ?? '', - requestBody: { enabled: e.detail } - }) - sendUserToast(`${e.detail ? 'enabled' : 'disabled'} schedule ${initialPath}`) - }} - /> -
- {/if} - - - -
- {#if !edit} -
- -
- {/if} -
- - Schedules use CRON syntax. Seconds are mandatory. - - -
-
- {#if !edit} -

- Pick a script or flow to be triggered by the schedule -

- - {:else} - - Once a schedule is created, the runnable path cannot be changed. However, when renaming - a script or a flow, the runnable path will automatically update itself. - -
- - {/if} -
- {#if runnable} - {#if runnable?.schema && runnable.schema.properties && Object.keys(runnable.schema.properties).length > 0} - - {:else} -
- This {is_flow ? 'flow' : 'script'} takes no argument -
- {/if} - {:else} -
- Pick a {is_flow ? 'flow' : 'script'} and fill its argument here -
- {/if} -
-
-
- - - -
-
The following args will be passed to the error handler: -
    -
  • path: The path of the script or flow that failed.
    • -
  • is_flow: Whether the runnable is a flow.
    • -
  • schedule_path: The path of the schedule.
    • -
  • error: The error details.
    • -
  • failed_times: Minimum number of times the schedule failed before calling - the error handler.
    • -
  • started_at: The start datetime of the latest job that failed.
    • -
-
-
- - - - -
-
-

{#if !$enterpriseLicense}(ee only){/if} Triggered - when schedule failed

- - -

time{failedTimes > 1 ? 's in a row' : ''}

-
-
-
-
- -
- {#if !$enterpriseLicense}(ee only){/if} -
- - - - - -
-
The following args will be passed to the recovery handler: -
    -
  • path: The path of the script or flow that recovered.
    • -
  • is_flow: Whether the runnable is a flow.
    • -
  • schedule_path: The path of the schedule.
    • -
  • error: The error of the last job that errored
    • -
  • error_started_at: The start datetime of the last job that errored
    • -
  • success_times: The number of times the schedule succeeded before calling - the recovery handler.
    • -
  • success_result: The result of the latest successful job
    • -
  • success_started_at: The start datetime of the latest successful job
    • -
-
-
- - - - -
-
-

Triggered when schedule recovered

- -

time{recoveredTimes > 1 ? 's in a row' : ''}

-
-
-
-
- - +{#if open} + +{/if} diff --git a/frontend/src/lib/components/ScheduleEditorInner.svelte b/frontend/src/lib/components/ScheduleEditorInner.svelte new file mode 100644 index 0000000000..e8715ed6d4 --- /dev/null +++ b/frontend/src/lib/components/ScheduleEditorInner.svelte @@ -0,0 +1,470 @@ + + + + + + {#if edit} +
+ +
+
+ { + await ScheduleService.setScheduleEnabled({ + path: initialPath, + workspace: $workspaceStore ?? '', + requestBody: { enabled: e.detail } + }) + sendUserToast(`${e.detail ? 'enabled' : 'disabled'} schedule ${initialPath}`) + }} + /> +
+ {/if} + + + +
+ {#if !edit} +
+ +
+ {/if} +
+ + Schedules use CRON syntax. Seconds are mandatory. + + +
+
+ {#if !edit} +

+ Pick a script or flow to be triggered by the schedule +

+ + {:else} + + Once a schedule is created, the runnable path cannot be changed. However, when renaming + a script or a flow, the runnable path will automatically update itself. + +
+ + {/if} +
+ {#if runnable} + {#if runnable?.schema && runnable.schema.properties && Object.keys(runnable.schema.properties).length > 0} + + {:else} +
+ This {is_flow ? 'flow' : 'script'} takes no argument +
+ {/if} + {:else} +
+ Pick a {is_flow ? 'flow' : 'script'} and fill its argument here +
+ {/if} +
+
+
+ + + +
+
The following args will be passed to the error handler: +
    +
  • path: The path of the script or flow that failed.
    • +
  • is_flow: Whether the runnable is a flow.
    • +
  • schedule_path: The path of the schedule.
    • +
  • error: The error details.
    • +
  • failed_times: Minimum number of times the schedule failed before + calling the error handler.
    • +
  • started_at: The start datetime of the latest job that failed.
    • +
+
+
+ + + + +
+
+

{#if !$enterpriseLicense}(ee only){/if} Triggered + when schedule failed

+ + +

time{failedTimes > 1 ? 's in a row' : ''}

+
+
+
+
+ +
+ {#if !$enterpriseLicense}(ee only){/if} +
+ + + + + +
+
The following args will be passed to the recovery handler: +
    +
  • path: The path of the script or flow that recovered.
    • +
  • is_flow: Whether the runnable is a flow.
    • +
  • schedule_path: The path of the schedule.
    • +
  • error: The error of the last job that errored
    • +
  • error_started_at: The start datetime of the last job that errored
    • +
  • success_times: The number of times the schedule succeeded before + calling the recovery handler.
    • +
  • success_result: The result of the latest successful job
    • +
  • success_started_at: The start datetime of the latest successful job
    • +
+
+
+ + + + +
+
+

Triggered when schedule recovered

+ +

time{recoveredTimes > 1 ? 's in a row' : ''}

+
+
+
+
+ + diff --git a/frontend/src/lib/components/TestJobLoader.svelte b/frontend/src/lib/components/TestJobLoader.svelte index 6bb628756b..35ab368bf3 100644 --- a/frontend/src/lib/components/TestJobLoader.svelte +++ b/frontend/src/lib/components/TestJobLoader.svelte @@ -157,6 +157,7 @@ job.mem_peak = previewJobUpdates.mem_peak } if ((previewJobUpdates.running ?? false) || (previewJobUpdates.completed ?? false)) { + console.log({ a: previewJobUpdates.running, b: previewJobUpdates.completed }) job = await JobService.getJob({ workspace: workspace!, id }) } } else { diff --git a/frontend/src/lib/components/WorkspaceGroup.svelte b/frontend/src/lib/components/WorkspaceGroup.svelte index 779e1b563a..6eced1cccc 100644 --- a/frontend/src/lib/components/WorkspaceGroup.svelte +++ b/frontend/src/lib/components/WorkspaceGroup.svelte @@ -45,7 +45,7 @@ 'other', 'bun' ] - const nativeTags = ['nativets', 'postgresql', 'mysql', 'graphql', 'snowflake'] + const nativeTags = ['nativets', 'postgresql', 'mysql', 'graphql', 'snowflake', 'bigquery'] let newTag: string = '' $: selected = nconfig?.dedicated_worker != undefined ? 'dedicated' : 'normal' diff --git a/frontend/src/lib/components/resourceTypesStore.ts b/frontend/src/lib/components/resourceTypesStore.ts index ab0d0063ce..2c2817d2f5 100644 --- a/frontend/src/lib/components/resourceTypesStore.ts +++ b/frontend/src/lib/components/resourceTypesStore.ts @@ -9,9 +9,18 @@ export async function getResourceTypes() { if (rts) { return rts } else { - const nrts = await ResourceService.listResourceTypeNames({ workspace: get(workspaceStore)! }) - resourceTypesStore.set(nrts) - return nrts + let workspace = get(workspaceStore) + if (workspace){ + try { + const nrts = await ResourceService.listResourceTypeNames({ workspace: workspace }) + resourceTypesStore.set(nrts) + return nrts + } catch (e) { + return ["error_fetching_names"] + } + } else { + return ['workspace_is_undefined'] + } } } diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 4dffac91e6..ba12e96dea 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -3,6 +3,7 @@ import { derived, type Readable, writable } from 'svelte/store' import type { UserWorkspaceList } from '$lib/gen/models/UserWorkspaceList.js' import type { TokenResponse } from './gen' import type { IntrospectionQuery } from 'graphql' +import { resourceTypesStore } from './components/resourceTypesStore' export interface UserExt { email: string @@ -108,6 +109,7 @@ export const dbSchemas = writable({}) export function switchWorkspace(workspace: string | undefined) { localStorage.removeItem('flow') localStorage.removeItem('app') + resourceTypesStore.set(undefined) workspaceStore.set(workspace) } @@ -115,6 +117,7 @@ export function clearStores(): void { localStorage.removeItem('flow') localStorage.removeItem('app') localStorage.removeItem('workspace') + resourceTypesStore.set(undefined) userStore.set(undefined) workspaceStore.set(undefined) usersWorkspaceStore.set(undefined) diff --git a/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte b/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte index 13b6b234f3..85ebf2f772 100644 --- a/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte +++ b/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte @@ -10,6 +10,9 @@ import Tooltip from '$lib/components/Tooltip.svelte' import FlowGraph from '$lib/components/graph/FlowGraph.svelte' import SchemaForm from '$lib/components/SchemaForm.svelte' + import { workspaceStore } from '$lib/stores' + + $workspaceStore = $page.params.workspace let job: Job | undefined = undefined let currentApprovers: { resume_id: number; approver: string }[] = [] @@ -145,7 +148,7 @@ {/if}
-

Flow arguments

+

Flow arguments