diff --git a/CHANGELOG.md b/CHANGELOG.md index c7d05a96ce..848d9a6f10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,35 @@ # Changelog +## [1.402.3](https://github.com/windmill-labs/windmill/compare/v1.402.2...v1.402.3) (2024-09-30) + + +### Bug Fixes + +* improve allowed domains setting for sso ([24f4a7c](https://github.com/windmill-labs/windmill/commit/24f4a7caaafa93f51669dcf44a3dca09d5b228bb)) + +## [1.402.2](https://github.com/windmill-labs/windmill/compare/v1.402.1...v1.402.2) (2024-09-28) + + +### Bug Fixes + +* make form properties disablable ([0779d47](https://github.com/windmill-labs/windmill/commit/0779d47c1d39626d11bd3769cd787cb036df0a94)) + +## [1.402.1](https://github.com/windmill-labs/windmill/compare/v1.402.0...v1.402.1) (2024-09-28) + + +### Bug Fixes + +* allow preprocessor to write to args.json on nsjail ([#4455](https://github.com/windmill-labs/windmill/issues/4455)) ([0b9ec83](https://github.com/windmill-labs/windmill/commit/0b9ec83036e2a1d0773b4ec5856f907b383e9323)) +* **frontend:** Fix flow graph bg in dark mode on chrome ([#4454](https://github.com/windmill-labs/windmill/issues/4454)) ([6956a3a](https://github.com/windmill-labs/windmill/commit/6956a3a2ba6d189528cb34ab05f7137fdf4f840b)) +* improve suspend_first behavior and frequency ([b5e226b](https://github.com/windmill-labs/windmill/commit/b5e226b977e6d24ebd28bc1e7c867cb4888f77b2)) + +## [1.402.0](https://github.com/windmill-labs/windmill/compare/v1.401.0...v1.402.0) (2024-09-26) + + +### Features + +* **cli:** add queues, workers and worker-groups commands ([#4439](https://github.com/windmill-labs/windmill/issues/4439)) ([9f91b19](https://github.com/windmill-labs/windmill/commit/9f91b1995a98c9e096c6e599c4d5a8d5ea499ada)) + ## [1.401.0](https://github.com/windmill-labs/windmill/compare/v1.400.0...v1.401.0) (2024-09-25) diff --git a/backend/.sqlx/query-1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe.json b/backend/.sqlx/query-1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe.json new file mode 100644 index 0000000000..01fb19b1c2 --- /dev/null +++ b/backend/.sqlx/query-1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe" +} diff --git a/backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json b/backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json new file mode 100644 index 0000000000..f193758557 --- /dev/null +++ b/backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838" +} diff --git a/backend/.sqlx/query-9bf41c3161a02b7d0731c4e1d79519cef5255f5df1b759af3aa4985bb64313e5.json b/backend/.sqlx/query-41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a.json similarity index 78% rename from backend/.sqlx/query-9bf41c3161a02b7d0731c4e1d79519cef5255f5df1b759af3aa4985bb64313e5.json rename to backend/.sqlx/query-41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a.json index 2eec9fba30..9397f52b83 100644 --- a/backend/.sqlx/query-9bf41c3161a02b7d0731c4e1d79519cef5255f5df1b759af3aa4985bb64313e5.json +++ b/backend/.sqlx/query-41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO metrics (id, value)\n VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n ORDER BY priority DESC NULLS LAST, scheduled_for, created_at LIMIT 1)))", + "query": "INSERT INTO metrics (id, value)\n VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "9bf41c3161a02b7d0731c4e1d79519cef5255f5df1b759af3aa4985bb64313e5" + "hash": "41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a" } diff --git a/backend/.sqlx/query-8375c1efeb1e2a2d2803052a2899bf70f4a6434eb91b4b05b9fb8420beae26af.json b/backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json similarity index 68% rename from backend/.sqlx/query-8375c1efeb1e2a2d2803052a2899bf70f4a6434eb91b4b05b9fb8420beae26af.json rename to backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json index 4bcbb0380f..9df7dced7b 100644 --- a/backend/.sqlx/query-8375c1efeb1e2a2d2803052a2899bf70f4a6434eb91b4b05b9fb8420beae26af.json +++ b/backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", + "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, \n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -65,21 +65,36 @@ }, { "ordinal": 12, + "name": "occupancy_rate_15s", + "type_info": "Float4" + }, + { + "ordinal": 13, + "name": "occupancy_rate_5m", + "type_info": "Float4" + }, + { + "ordinal": 14, + "name": "occupancy_rate_30m", + "type_info": "Float4" + }, + { + "ordinal": 15, "name": "memory", "type_info": "Int8" }, { - "ordinal": 13, + "ordinal": 16, "name": "vcpus", "type_info": "Int8" }, { - "ordinal": 14, + "ordinal": 17, "name": "memory_usage", "type_info": "Int8" }, { - "ordinal": 15, + "ordinal": 18, "name": "wm_memory_usage", "type_info": "Int8" } @@ -108,8 +123,11 @@ true, true, true, + true, + true, + true, true ] }, - "hash": "8375c1efeb1e2a2d2803052a2899bf70f4a6434eb91b4b05b9fb8420beae26af" + "hash": "6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab" } diff --git a/backend/.sqlx/query-6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46.json b/backend/.sqlx/query-6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46.json new file mode 100644 index 0000000000..e8db78adbd --- /dev/null +++ b/backend/.sqlx/query-6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46" +} diff --git a/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json b/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json new file mode 100644 index 0000000000..5605518c9c --- /dev/null +++ b/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json @@ -0,0 +1,73 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, raw_flow, flow_status) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 FROM generate_series(1, 1))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Varchar", + { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlescriptflow" + ] + } + } + }, + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible" + ] + } + } + }, + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Timestamptz", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561" +} diff --git a/backend/.sqlx/query-a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c.json b/backend/.sqlx/query-777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0.json similarity index 60% rename from backend/.sqlx/query-a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c.json rename to backend/.sqlx/query-777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0.json index 5906375f53..0c1d65bb6b 100644 --- a/backend/.sqlx/query-a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c.json +++ b/backend/.sqlx/query-777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb)\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", "describe": { "columns": [ { @@ -19,5 +19,5 @@ null ] }, - "hash": "a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c" + "hash": "777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0" } diff --git a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json new file mode 100644 index 0000000000..c4a2ff0a10 --- /dev/null +++ b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json @@ -0,0 +1,45 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar", + "Varchar", + "Int8", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible" + ] + } + } + }, + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf" +} diff --git a/backend/.sqlx/query-9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51.json b/backend/.sqlx/query-9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51.json deleted file mode 100644 index 92825535a4..0000000000 --- a/backend/.sqlx/query-9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), memory = COALESCE($8, memory) WHERE worker = $6", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "TextArray", - "Float4", - "Int8", - "Int8", - "Text", - "Int8", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51" -} diff --git a/backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json b/backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json new file mode 100644 index 0000000000..691a6d5e31 --- /dev/null +++ b/backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "TextArray", + "Float4", + "Int8", + "Int8", + "Text", + "Int8", + "Int8", + "Float4", + "Float4", + "Float4" + ] + }, + "nullable": [] + }, + "hash": "a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb" +} diff --git a/backend/.sqlx/query-ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4.json b/backend/.sqlx/query-ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4.json deleted file mode 100644 index 74c890726f..0000000000 --- a/backend/.sqlx/query-ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT raw_flow->'modules'->$2::int->'retry' FROM queue WHERE id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Int4" - ] - }, - "nullable": [ - null - ] - }, - "hash": "ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4" -} diff --git a/backend/.sqlx/query-c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb.json b/backend/.sqlx/query-c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb.json new file mode 100644 index 0000000000..b545c48599 --- /dev/null +++ b/backend/.sqlx/query-c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT tag as \"tag!\", COUNT(*) as \"count!\"\n FROM completed_job\n WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)\n GROUP BY tag\n ORDER BY \"count!\" DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb" +} diff --git a/backend/.sqlx/query-d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12.json b/backend/.sqlx/query-d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12.json deleted file mode 100644 index 29227c3d8c..0000000000 --- a/backend/.sqlx/query-d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET last_ping = null\n WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12" -} diff --git a/backend/.sqlx/query-d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43.json b/backend/.sqlx/query-d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43.json deleted file mode 100644 index 7d7274d774..0000000000 --- a/backend/.sqlx/query-d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43" -} diff --git a/backend/.sqlx/query-95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4.json b/backend/.sqlx/query-e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c.json similarity index 50% rename from backend/.sqlx/query-95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4.json rename to backend/.sqlx/query-e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c.json index 991be7c977..3bacda16d9 100644 --- a/backend/.sqlx/query-95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4.json +++ b/backend/.sqlx/query-e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4 WHERE worker = $5", + "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5", "describe": { "columns": [], "parameters": { @@ -9,10 +9,14 @@ "Varchar", "Int8", "Int8", - "Text" + "Text", + "Float4", + "Float4", + "Float4", + "Float4" ] }, "nullable": [] }, - "hash": "95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4" + "hash": "e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c" } diff --git a/backend/.sqlx/query-f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b.json b/backend/.sqlx/query-f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b.json new file mode 100644 index 0000000000..2cf1fb31e9 --- /dev/null +++ b/backend/.sqlx/query-f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n ),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7c71bd74da..0ee755b3ec 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ "memchr", "num", "regex", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -465,7 +465,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -612,7 +612,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -660,7 +660,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -702,7 +702,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -750,15 +750,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.6" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848d7b9b605720989929279fa644ce8f244d0ce3146fcca5b70e4eb7b3c020fc" +checksum = "8191fb3091fa0561d1379ef80333c3c7191c6f0435d986e85821bcf7acbd1126" dependencies = [ "aws-credential-types", "aws-runtime", @@ -823,9 +823,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.43.0" +version = "1.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a9d27ed1c12b1140c47daf1bc541606c43fdafd918c4797d520db0043ceef2" +checksum = "0b90cfe6504115e13c41d3ea90286ede5aa14da294f3fe077027a6e83850843c" dependencies = [ "aws-credential-types", "aws-runtime", @@ -845,9 +845,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.44.0" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44514a6ca967686cde1e2a1b81df6ef1883d0e3e570da8d8bc5c491dcb6fc29b" +checksum = "167c0fad1f212952084137308359e8e4c4724d1c643038ce163f06de9662c1d0" dependencies = [ "aws-credential-types", "aws-runtime", @@ -867,9 +867,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.43.0" +version = "1.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7a4d279762a35b9df97209f6808b95d4fe78547fe2316b4d200a0283960c5a" +checksum = "2cb5f98188ec1435b68097daa2a37d74b9d17c9caa799466338a8d1544e71b9d" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1007,9 +1007,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.6" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03701449087215b5369c7ea17fef0dd5d24cb93439ec5af0c7615f58c3f22605" +checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" dependencies = [ "base64-simd 0.8.0", "bytes", @@ -1056,9 +1056,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec" +checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" dependencies = [ "async-trait", "axum-core", @@ -1091,9 +1091,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", @@ -1242,7 +1242,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.77", + "syn 2.0.79", "which 4.4.2", ] @@ -1261,7 +1261,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1405,7 +1405,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "syn_derive", ] @@ -1494,7 +1494,7 @@ checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1609,9 +1609,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.21" +version = "1.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" dependencies = [ "jobserver", "libc", @@ -1764,7 +1764,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -2150,7 +2150,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -2198,7 +2198,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -2220,7 +2220,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -2454,7 +2454,7 @@ dependencies = [ "indexmap 2.5.0", "itertools 0.12.1", "log", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -2740,7 +2740,7 @@ dependencies = [ "quote", "strum 0.25.0", "strum_macros 0.25.3", - "syn 2.0.77", + "syn 2.0.79", "thiserror", ] @@ -2919,7 +2919,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3016,7 +3016,7 @@ checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3172,7 +3172,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3192,7 +3192,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3331,9 +3331,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.33" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", "libz-sys", @@ -3395,7 +3395,7 @@ checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" dependencies = [ "proc-macro2", "swc_macros_common", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3427,7 +3427,7 @@ checksum = "e99b8b3c28ae0e84b604c75f721c21dc77afb3706076af5e8216d15fd1deaae3" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3439,7 +3439,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3451,7 +3451,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3575,7 +3575,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3828,7 +3828,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4461,7 +4461,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4710,7 +4710,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", - "redox_syscall 0.5.5", + "redox_syscall 0.5.7", ] [[package]] @@ -5112,7 +5112,7 @@ checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -5151,7 +5151,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "termcolor", "thiserror", ] @@ -5457,9 +5457,12 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" +dependencies = [ + "portable-atomic", +] [[package]] name = "oneshot" @@ -5561,7 +5564,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -5672,7 +5675,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.5", + "redox_syscall 0.5.7", "smallvec", "windows-targets 0.52.6", ] @@ -5879,7 +5882,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -5920,7 +5923,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6012,9 +6015,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" [[package]] name = "postgres-native-tls" @@ -6102,7 +6105,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6155,7 +6158,7 @@ checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" dependencies = [ "proc-macro-rules-macros", "proc-macro2", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6167,7 +6170,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6228,7 +6231,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "syn 2.0.77", + "syn 2.0.79", "thiserror", "typify", "unicode-ident", @@ -6248,7 +6251,7 @@ dependencies = [ "serde_json", "serde_tokenstream", "serde_yaml", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6291,7 +6294,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -6623,9 +6626,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] @@ -6643,14 +6646,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -6664,13 +6667,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -6693,9 +6696,9 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "regress" @@ -6974,7 +6977,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.77", + "syn 2.0.79", "walkdir", ] @@ -7163,9 +7166,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" [[package]] name = "rustls-tokio-stream" @@ -7355,7 +7358,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -7502,7 +7505,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -7513,7 +7516,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -7572,9 +7575,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -7588,7 +7591,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -7643,7 +7646,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -7993,7 +7996,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8064,7 +8067,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8087,7 +8090,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.77", + "syn 2.0.79", "tempfile", "tokio", "url", @@ -8239,7 +8242,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8293,7 +8296,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8306,7 +8309,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8425,7 +8428,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8491,7 +8494,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8598,7 +8601,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8717,7 +8720,7 @@ checksum = "695a1d8b461033d32429b5befbf0ad4d7a2c4d6ba9cd5ba4e0645c615839e8e4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8728,7 +8731,7 @@ checksum = "91745f3561057493d2da768437c427c0e979dff7396507ae02f16c981c4a8466" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8751,7 +8754,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8767,9 +8770,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.77" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -8785,7 +8788,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8811,7 +8814,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -8966,7 +8969,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" dependencies = [ "byteorder", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", "utf8-ranges", ] @@ -9030,9 +9033,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand 2.1.1", @@ -9076,7 +9079,7 @@ checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -9300,7 +9303,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -9468,7 +9471,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.19", + "winnow 0.6.20", ] [[package]] @@ -9597,7 +9600,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -9815,7 +9818,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.77", + "syn 2.0.79", "thiserror", "unicode-ident", ] @@ -9832,7 +9835,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.77", + "syn 2.0.79", "typify-impl", ] @@ -10225,7 +10228,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "wasm-bindgen-shared", ] @@ -10259,7 +10262,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -10292,14 +10295,14 @@ checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "wasm-streams" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" dependencies = [ "futures-util", "js-sys", @@ -10382,7 +10385,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "redox_syscall 0.5.5", + "redox_syscall 0.5.7", "wasite", "web-sys", ] @@ -10426,7 +10429,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "axum", @@ -10468,7 +10471,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "argon2", @@ -10553,7 +10556,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.401.0" +version = "1.402.3" dependencies = [ "base64 0.21.7", "chrono", @@ -10571,7 +10574,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.401.0" +version = "1.402.3" dependencies = [ "chrono", "serde", @@ -10584,7 +10587,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "async-stream", @@ -10628,7 +10631,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.401.0" +version = "1.402.3" dependencies = [ "regex", "rsmq_async", @@ -10643,7 +10646,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "bytes", @@ -10664,7 +10667,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.401.0" +version = "1.402.3" dependencies = [ "convert_case 0.6.0", "serde", @@ -10673,7 +10676,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "lazy_static", @@ -10685,7 +10688,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "gosyn", @@ -10697,7 +10700,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "lazy_static", @@ -10709,7 +10712,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10720,7 +10723,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10731,7 +10734,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "async-recursion", @@ -10749,7 +10752,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10759,14 +10762,14 @@ dependencies = [ "quote", "regex", "serde_json", - "syn 2.0.77", + "syn 2.0.79", "toml 0.7.8", "windmill-parser", ] [[package]] name = "windmill-parser-sql" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "lazy_static", @@ -10778,7 +10781,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "lazy_static", @@ -10796,7 +10799,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10817,7 +10820,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "serde_json", @@ -10827,7 +10830,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "async-recursion", @@ -10860,7 +10863,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.401.0" +version = "1.402.3" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10870,7 +10873,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.401.0" +version = "1.402.3" dependencies = [ "anyhow", "async-recursion", @@ -11142,9 +11145,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.19" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -11235,7 +11238,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "synstructure", ] @@ -11257,7 +11260,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -11277,7 +11280,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "synstructure", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 9a657a683f..9b7a0123c8 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.401.0" +version = "1.402.3" authors.workspace = true edition.workspace = true @@ -27,7 +27,7 @@ members = [ ] [workspace.package] -version = "1.401.0" +version = "1.402.3" authors = ["Ruben Fiszel "] edition = "2021" @@ -44,7 +44,7 @@ default = [] enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"] enterprise_saml = ["windmill-api/enterprise_saml"] stripe = ["windmill-api/stripe"] -benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"] +benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"] flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"] loki = ["windmill-common/loki"] pg_embed = ["dep:pg-embed"] diff --git a/backend/migrations/20240924213010_add_last_occupancy_rates.down.sql b/backend/migrations/20240924213010_add_last_occupancy_rates.down.sql new file mode 100644 index 0000000000..d2249d992d --- /dev/null +++ b/backend/migrations/20240924213010_add_last_occupancy_rates.down.sql @@ -0,0 +1,5 @@ +-- Add down migration script here +ALTER TABLE worker_ping +DROP COLUMN occupancy_rate_15s, +DROP COLUMN occupancy_rate_5m, +DROP COLUMN occupancy_rate_30m; diff --git a/backend/migrations/20240924213010_add_last_occupancy_rates.up.sql b/backend/migrations/20240924213010_add_last_occupancy_rates.up.sql new file mode 100644 index 0000000000..39033324c3 --- /dev/null +++ b/backend/migrations/20240924213010_add_last_occupancy_rates.up.sql @@ -0,0 +1,5 @@ +-- Add up migration script here +ALTER TABLE worker_ping +ADD COLUMN occupancy_rate_15s REAL, +ADD COLUMN occupancy_rate_5m REAL, +ADD COLUMN occupancy_rate_30m REAL; diff --git a/backend/plot.py b/backend/plot.py new file mode 100644 index 0000000000..2b02984a62 --- /dev/null +++ b/backend/plot.py @@ -0,0 +1,89 @@ +import json +import matplotlib.pyplot as plt + +# Function to load JSON data from a file +def load_json_data(filepath): + with open(filepath, 'r') as file: + data = json.load(file) + return data + +# Function to plot two arrays of subarrays with tuples (step_name, duration) +def plot_two_arrays_of_subarrays(arrays1, arrays2): + # Function to calculate sum of durations for each step + def calculate_sums(arrays): + steps = [step for step, _ in arrays[0]['timings']] # Extract steps from the first iteration + sums = {step: 0.0 for step in steps} # Initialize sums dictionary with step names + + # Sum up the durations for each step across all subarrays + for subarray in arrays: + for step_name, duration in subarray['timings']: + if step_name not in sums: + sums[step_name] = 0 + sums[step_name] += duration + + for step_name, duration in sums.items(): + sums[step_name] = duration / 1000000000 + + # Convert the sums dictionary to two lists (for plotting) + step_names = list(sums.keys()) + durations = list(sums.values()) + return step_names, durations + + # Calculate sums for both arrays of subarrays + step_names1, sums1 = calculate_sums(arrays1) + step_names2, sums2 = calculate_sums(arrays2) + + # Create two subplots, one on top of the other + fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 12)) + + # First plot (top) for the first array of subarrays + ax1.bar(step_names1, sums1, color='b') + ax1.set_title('Total Duration per Step - Main Loop') + ax1.set_xlabel('Step Name') + ax1.set_ylabel('Total Duration (s)') + ax1.grid(True, axis='y') + ax1.tick_params(axis='x', rotation=45) + + # Second plot (bottom) for the second array of subarrays + ax2.bar(step_names2, sums2, color='r') + ax2.set_title('Total Duration per Step - Result Processor') + ax2.set_xlabel('Step Name') + ax2.set_ylabel('Total Duration (s)') + ax2.grid(True, axis='y') + ax2.tick_params(axis='x', rotation=45) + + # Adjust layout so the plots don't overlap + plt.tight_layout() + + # Display the plot + plt.show() + +# Load arrays from the JSON files +main = load_json_data('/tmp/windmill/profiling_main.json') +result_processor = load_json_data('/tmp/windmill/profiling_result_processor.json') + +arrays1 = main['timings'] +arrays2 = result_processor['timings'] + +total_duration1 = main['total_duration']/1000 +total_duration2 = result_processor['total_duration']/1000 + +print(f"Total duration for main: {total_duration1}s") +print(f"Total duration for result processor: {total_duration2}s") + +iterations_total = sum(main['iter_durations']) / 1000000000 +iterations_total2 = sum(result_processor['iter_durations']) / 1000000000 + +print(f"Number of iterations: {len(main['iter_durations'])}") +print(f"Total iterations for main: {iterations_total}s") +print(f"Total iterations for result processor: {iterations_total2}s") + +# Calculate RPS +rps1 = len(main['iter_durations']) / total_duration1 +rps2 = len(result_processor['iter_durations']) / total_duration2 + +print(f"RPS for main: {rps1}") +print(f"RPS for result processor: {rps2}") + +# Plot the data +plot_two_arrays_of_subarrays(arrays1, arrays2) \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index 20bc963bd5..aa6993cf92 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -158,6 +158,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { "global", "global", "", + &mut None, ) .await?; tokio::fs::remove_dir_all(job_dir).await?; @@ -178,6 +179,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { "cache_init", windmill_worker::get_common_bun_proc_envs(None).await, false, + &mut None, ) .await?; } else { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 8397f134aa..92f69a0d64 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -525,7 +525,7 @@ fn read_log_counters(ts_str: String) -> (usize, usize) { ok_lines = counter.non_error_count; err_lines = counter.error_count; } else { - println!("no counter found for {ts_str}"); + // println!("no counter found for {ts_str}"); } } else { println!("Error reading log counters 2"); @@ -1071,73 +1071,68 @@ pub async fn monitor_db( } pub async fn expose_queue_metrics(db: &Pool) { - let tx = db.begin().await; - if let Ok(mut tx) = tx { - let last_check = sqlx::query_scalar!( + let last_check = sqlx::query_scalar!( "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1" ) .fetch_optional(db) .await .unwrap_or(Some(chrono::Utc::now())); - let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed); - let save_metrics = last_check - .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25)) - .unwrap_or(true); + let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed); + let save_metrics = last_check + .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25)) + .unwrap_or(true); - if metrics_enabled || save_metrics { - let queue_counts = sqlx::query!( - "SELECT tag, count(*) as count FROM queue WHERE + if metrics_enabled || save_metrics { + let queue_counts = sqlx::query!( + "SELECT tag, count(*) as count FROM queue WHERE scheduled_for <= now() - ('3 seconds')::interval AND running = false GROUP BY tag" - ) - .fetch_all(&mut *tx) - .await - .ok() - .unwrap_or_else(|| vec![]); + ) + .fetch_all(db) + .await + .ok() + .unwrap_or_else(|| vec![]); - for q in queue_counts { - let count = q.count.unwrap_or(0); - let tag = q.tag; - if metrics_enabled { - let metric = (*QUEUE_COUNT).with_label_values(&[&tag]); - metric.set(count as i64); - } + for q in queue_counts { + let count = q.count.unwrap_or(0); + let tag = q.tag; + if metrics_enabled { + let metric = (*QUEUE_COUNT).with_label_values(&[&tag]); + metric.set(count as i64); + } - // save queue_count and delay metrics per tag - if save_metrics { + // save queue_count and delay metrics per tag + if save_metrics { + sqlx::query!( + "INSERT INTO metrics (id, value) VALUES ($1, $2)", + format!("queue_count_{}", tag), + serde_json::json!(count) + ) + .execute(db) + .await + .ok(); + if count > 0 { sqlx::query!( - "INSERT INTO metrics (id, value) VALUES ($1, $2)", - format!("queue_count_{}", tag), - serde_json::json!(count) - ) - .execute(&mut *tx) - .await - .ok(); - if count > 0 { - sqlx::query!( "INSERT INTO metrics (id, value) VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for) FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval - ORDER BY priority DESC NULLS LAST, scheduled_for, created_at LIMIT 1)))", + ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))", format!("queue_delay_{}", tag), tag - ).execute(&mut *tx).await.ok(); - } + ).execute(db).await.ok(); } } } - - // clean queue metrics older than 14 days - sqlx::query!( - "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'" - ) - .execute(&mut *tx) - .await - .ok(); - - tx.commit().await.ok(); } + + // clean queue metrics older than 14 days + sqlx::query!( + "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'" + ) + .execute(db) + .await + .ok(); } pub async fn reload_smtp_config(db: &Pool) { @@ -1373,6 +1368,8 @@ async fn handle_zombie_jobs rsmq.clone(), worker_name, send_result_never_used, + #[cfg(feature = "benchmark")] + &mut windmill_common::bench::BenchmarkIter::new(), ) .await; } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 3245b03ed5..c8c30a7db1 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.401.0 + version: 1.402.3 title: Windmill API contact: @@ -5964,6 +5964,43 @@ paths: schema: type: integer + /jobs/completed/count_by_tag: + get: + summary: Count jobs by tag + operationId: countJobsByTag + tags: + - job + parameters: + - name: horizon_secs + in: query + description: Past Time horizon in seconds (when to start the count = now - horizon) (default is 3600) + required: false + schema: + type: integer + - name: workspace_id + in: query + description: Specific workspace ID to filter results (optional) + required: false + schema: + type: string + responses: + "200": + description: Job counts by tag + content: + application/json: + schema: + type: array + items: + type: object + properties: + tag: + type: string + count: + type: integer + required: + - tag + - count + /w/{workspace}/jobs_u/get/{id}: get: summary: get job @@ -6498,6 +6535,7 @@ paths: schema: type: string + /w/{workspace}/jobs_u/cancel/{id}/{resume_id}/{signature}: get: summary: cancel a job for a suspended flow @@ -9949,6 +9987,8 @@ components: type: number aggregate_wait_time_ms: type: number + suspend: + type: number required: - id - running @@ -11158,6 +11198,12 @@ components: type: string occupancy_rate: type: number + occupancy_rate_15s: + type: number + occupancy_rate_5m: + type: number + occupancy_rate_30m: + type: number memory: type: number vcpus: diff --git a/backend/windmill-api/src/configs.rs b/backend/windmill-api/src/configs.rs index aacfd49ef7..64c8a1884f 100644 --- a/backend/windmill-api/src/configs.rs +++ b/backend/windmill-api/src/configs.rs @@ -33,7 +33,7 @@ pub fn global_service() -> Router { #[derive(Serialize, Deserialize, FromRow)] struct Config { - name: String, + name: Option, config: serde_json::Value, } @@ -41,9 +41,18 @@ async fn list_worker_groups( authed: ApiAuthed, Extension(db): Extension, ) -> error::JsonResult> { - let configs_raw = sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'") - .fetch_all(&db) - .await?; + let mut configs_raw = + sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'") + .fetch_all(&db) + .await?; + // Remove the 'worker__' prefix from all config names + for config in configs_raw.iter_mut() { + if let Some(name) = &config.name { + if name.starts_with("worker__") { + config.name = Some(name.strip_prefix("worker__").unwrap().to_string()); + } + } + } let configs = if !authed.is_admin { let mut obfuscated_configs: Vec = vec![]; for config in configs_raw { diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 6e74866203..0b5082c6e0 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -242,6 +242,7 @@ pub async fn get_hub_flow_by_id( #[derive(Deserialize)] pub struct ToggleWorkspaceErrorHandler { + #[cfg(feature = "enterprise")] pub muted: Option, } diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index c43d2935e3..156be41b26 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -11,6 +11,7 @@ use axum::http::HeaderValue; use quick_cache::sync::Cache; use serde_json::value::RawValue; use sqlx::Pool; +use windmill_common::error::JsonResult; use std::collections::HashMap; #[cfg(feature = "prometheus")] use std::sync::atomic::Ordering; @@ -69,7 +70,7 @@ use windmill_common::{ oauth2::HmacSha256, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, - utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath}, + utils::{not_found_if_none, now_from_db, paginate, paginate_without_limits, require_admin, Pagination, StripPath}, }; #[cfg(all(feature = "enterprise", feature = "parquet"))] @@ -247,7 +248,7 @@ pub fn workspaced_service() -> Router { .route("/run/flow_dependencies", post(run_flow_dependencies_job)) } -pub fn global_service() -> Router { +pub fn workspace_unauthed_service() -> Router { Router::new() .route( "/resume/:job_id/:resume_id/:secret", @@ -291,7 +292,12 @@ pub fn global_service() -> Router { } pub fn global_root_service() -> Router { - Router::new().route("/db_clock", get(get_db_clock)) + Router::new() + .route("/db_clock", get(get_db_clock)) + .route( + "/completed/count_by_tag", + get(count_by_tag), + ) } #[derive(Deserialize)] @@ -1248,13 +1254,16 @@ pub fn list_queue_jobs_query( w_id: &str, lq: &ListQueueQuery, fields: &[&str], + pagination: Pagination, join_outstanding_wait_times: bool, tags: Option>, ) -> SqlBuilder { + let (limit, offset) = paginate_without_limits(pagination); let mut sqlb = SqlBuilder::select_from("queue") .fields(fields) .order_by("created_at", lq.order_desc.unwrap_or(true)) - .limit(1000) + .limit(limit) + .offset(offset) .clone(); if let Some(tags) = tags { @@ -1267,6 +1276,7 @@ pub fn list_queue_jobs_query( #[derive(Serialize, FromRow)] struct ListableQueuedJob { pub id: Uuid, + pub running: bool, pub created_by: String, pub created_at: chrono::DateTime, pub started_at: Option>, @@ -1289,6 +1299,7 @@ async fn list_queue_jobs( authed: ApiAuthed, Extension(user_db): Extension, Path(w_id): Path, + Query(pagination): Query, Query(lq): Query, ) -> error::JsonResult> { let sql = list_queue_jobs_query( @@ -1296,6 +1307,7 @@ async fn list_queue_jobs( &lq, &[ "id", + "running", "created_by", "created_at", "started_at", @@ -1315,6 +1327,7 @@ async fn list_queue_jobs( "priority", "workspace_id", ], + pagination, false, get_scope_tags(&authed), ) @@ -1570,6 +1583,7 @@ async fn list_jobs( ) -> error::JsonResult> { check_scopes(&authed, || format!("jobs:listjobs"))?; + let limit = pagination.per_page.unwrap_or(1000); let (per_page, offset) = paginate(pagination); let lqc = lq.clone(); @@ -1601,6 +1615,7 @@ async fn list_jobs( &w_id, &ListQueueQuery { order_desc: Some(true), ..lq.into() }, UnifiedJob::queued_job_fields(), + Pagination { per_page: Some(limit), page: None }, true, get_scope_tags(&authed), ); @@ -5100,6 +5115,46 @@ async fn get_completed_job_result( Ok(Json(result).into_response()) } + + +#[derive(Deserialize)] +struct CountByTagQuery { + horizon_secs: Option, + workspace_id: Option, +} + +#[derive(Serialize)] +struct TagCount { + tag: String, + count: i64, +} + +async fn count_by_tag( + ApiAuthed { email, ..}: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult> { + require_super_admin(&db, &email).await?; + let horizon = query.horizon_secs.unwrap_or(3600); // Default to 1 hour if not specified + + let counts = sqlx::query_as!( + TagCount, + r#" + SELECT tag as "tag!", COUNT(*) as "count!" + FROM completed_job + WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2) + GROUP BY tag + ORDER BY "count!" DESC + "#, + horizon as f64, + query.workspace_id + ) + .fetch_all(&db) + .await?; + + Ok(Json(counts)) +} + #[derive(Serialize)] struct CompletedJobResult { started: Option, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index b5bba9f645..97d2865e2e 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -330,7 +330,7 @@ pub async fn run_server( ) .nest( "/w/:workspace_id/jobs_u", - jobs::global_service().layer(cors.clone()), + jobs::workspace_unauthed_service().layer(cors.clone()), ) .nest( "/w/:workspace_id/resources_u", diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index aedb16bb15..bcede4f5e2 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -967,6 +967,7 @@ async fn list_paths( #[derive(Deserialize)] pub struct ToggleWorkspaceErrorHandler { + #[cfg(feature = "enterprise")] pub muted: Option, } diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index eecefe6146..2f8439bf24 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -54,6 +54,12 @@ struct WorkerPing { #[serde(skip_serializing_if = "Option::is_none")] occupancy_rate: Option, #[serde(skip_serializing_if = "Option::is_none")] + occupancy_rate_15s: Option, + #[serde(skip_serializing_if = "Option::is_none")] + occupancy_rate_5m: Option, + #[serde(skip_serializing_if = "Option::is_none")] + occupancy_rate_30m: Option, + #[serde(skip_serializing_if = "Option::is_none")] memory: Option, #[serde(skip_serializing_if = "Option::is_none")] vcpus: Option, @@ -88,7 +94,9 @@ async fn list_worker_pings( let rows = sqlx::query_as!( WorkerPing, - "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage + "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, + CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, + custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage FROM worker_ping WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval) ORDER BY ping_at desc LIMIT $2 OFFSET $3", diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 685286d750..95cd97fad0 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -42,7 +42,10 @@ use windmill_common::schedule::Schedule; use windmill_common::users::username_to_permissioned_as; use windmill_common::variables::build_crypt; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; -use windmill_common::workspaces::{WorkspaceDeploymentUISettings, WorkspaceGitSyncSettings}; +#[cfg(feature = "enterprise")] +use windmill_common::workspaces::WorkspaceDeploymentUISettings; +#[cfg(feature = "enterprise")] +use windmill_common::workspaces::WorkspaceGitSyncSettings; use windmill_common::{ error::{to_anyhow, Error, JsonResult, Result}, flows::Flow, @@ -991,6 +994,7 @@ async fn edit_large_file_storage_config( #[derive(Deserialize)] pub struct EditGitSyncConfig { + #[cfg(feature = "enterprise")] pub git_sync_settings: Option, } @@ -1056,6 +1060,7 @@ async fn edit_git_sync_config( #[derive(Deserialize)] struct EditDeployUIConfig { + #[cfg(feature = "enterprise")] deploy_ui_settings: Option, } @@ -1064,7 +1069,6 @@ async fn edit_deploy_ui_config( _authed: ApiAuthed, Extension(_db): Extension, Path(_w_id): Path, - Json(_new_config): Json, ) -> Result { return Err(Error::BadRequest( "Deployment UI is only available on Windmill Enterprise Edition".to_string(), @@ -1122,6 +1126,7 @@ async fn edit_deploy_ui_config( #[derive(Deserialize)] pub struct EditDefaultApp { + #[cfg(feature = "enterprise")] pub default_app_path: Option, } diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 5f45e051d4..09acf22157 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -11,6 +11,7 @@ jemalloc = ["dep:tikv-jemalloc-ctl"] prometheus = ["dep:prometheus"] flamegraph = ["dep:tracing-flame"] loki = ["dep:tracing-loki"] +benchmark = [] parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"] [lib] diff --git a/backend/windmill-common/src/bench.rs b/backend/windmill-common/src/bench.rs new file mode 100644 index 0000000000..0e89af571c --- /dev/null +++ b/backend/windmill-common/src/bench.rs @@ -0,0 +1,213 @@ +use crate::{ + worker::{write_file, TMP_DIR}, + DB, +}; +use serde::Serialize; +use tokio::time::Instant; + +#[derive(Serialize)] +pub struct BenchmarkInfo { + #[serde(skip)] + pub start: Instant, + #[serde(skip)] + pub iters: u64, + timings: Vec, + pub iter_durations: Vec, + pub total_duration: Option, +} + +impl BenchmarkInfo { + pub fn new() -> Self { + BenchmarkInfo { + iters: 0, + timings: vec![], + start: Instant::now(), + iter_durations: vec![], + total_duration: None, + } + } + + pub fn add_iter(&mut self, bench: BenchmarkIter, inc_iters: bool) { + if inc_iters { + self.iters += 1; + } + let elapsed_total = bench.start.elapsed().as_nanos() as u64; + self.timings.push(bench); + self.iter_durations.push(elapsed_total); + } + + pub fn write_to_file(&mut self, path: &str) -> anyhow::Result<()> { + let total_duration = self.start.elapsed().as_millis() as u64; + self.total_duration = Some(total_duration as u64); + + println!( + "Writing benchmark {path}, duration of benchmark: {total_duration}s and RPS: {}", + self.iters as f64 / total_duration as f64 + ); + write_file(TMP_DIR, path, &serde_json::to_string(&self).unwrap()).expect("write profiling"); + Ok(()) + } +} + +#[derive(Serialize)] +pub struct BenchmarkIter { + #[serde(skip)] + pub start: Instant, + #[serde(skip)] + last_instant: Instant, + last_step: String, + timings: Vec<(String, u32)>, +} + +impl BenchmarkIter { + pub fn new() -> Self { + BenchmarkIter { + last_instant: Instant::now(), + timings: vec![], + start: Instant::now(), + last_step: String::new(), + } + } + + pub fn add_timing(&mut self, name: &str) { + let elapsed = self.last_instant.elapsed().as_nanos() as u32; + self.timings + .push((format!("{}->{}", self.last_step, name), elapsed)); + self.last_instant = Instant::now(); + self.last_step = name.to_string(); + } +} + +pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) { + use crate::{jobs::JobKind, scripts::ScriptLang}; + + let benchmark_kind = std::env::var("BENCHMARK_KIND").unwrap_or("noop".to_string()); + + if benchmark_jobs > 0 { + match benchmark_kind.as_str() { + "dedicated" => { + // you need to create the script first, check https://github.com/windmill-labs/windmill/blob/b76a92cfe454c686f005c65f534e29e039f3c706/benchmarks/lib.ts#L47 + let hash = sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2", + "f/benchmarks/dedicated", + "admins" + ) + .fetch_one(db) + .await + .unwrap_or_else(|_e| panic!("failed to insert dedicated jobs")); + sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))", + hash, + "f/benchmarks/dedicated", + JobKind::Script as JobKind, + ScriptLang::Bun as ScriptLang, + "admins:f/benchmarks/dedicated", + "admin", + "u/admin", + "admin@windmill.dev", + chrono::Utc::now(), + "admins", + benchmark_jobs + ) + .execute(db) + .await.unwrap_or_else(|_e| panic!("failed to insert dedicated jobs")); + } + "parallelflow" => { + //create dedicated script + sqlx::query!("INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING", + "export async function main() { + console.log('hello world'); + }", + "admins", + "u/admin/parallelflow", + 1234567890, + ScriptLang::Deno as ScriptLang, + "flow", + "admin", + ) + .execute(db) + .await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs {_e:#}")); + sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, raw_flow, flow_status) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 FROM generate_series(1, 1))", + None::, + None::, + JobKind::FlowPreview as JobKind, + ScriptLang::Deno as ScriptLang, + "flow", + "admin", + "u/admin", + "admin@windmill.dev", + chrono::Utc::now(), + "admins", + serde_json::from_str::(r#" +{ + "modules": [ + { + "id": "a", + "value": { + "type": "forloopflow", + "modules": [ + { + "id": "b", + "value": { + "path": "u/admin/parallelflow", + "type": "script", + "tag_override": "", + "input_transforms": {} + }, + "summary": "calctest" + } + ], + "iterator": { + "expr": "[...new Array(300)]", + "type": "javascript" + }, + "parallel": true, + "parallelism": 10, + "skip_failures": true + } + } + ], + "preprocessor_module": null +} + "#).unwrap(), + serde_json::from_str::(r#" +{ + "step": 0, + "modules": [ + { + "id": "a", + "type": "WaitingForPriorSteps" + } + ], + "cleanup_module": {}, + "failure_module": { + "id": "failure", + "type": "WaitingForPriorSteps" + }, + "preprocessor_module": null + } + + "#).unwrap() + ) + .execute(db) + .await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs")); + } + _ => { + sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))", + None::, + None::, + JobKind::Noop as JobKind, + ScriptLang::Deno as ScriptLang, + "deno", + "admin", + "u/admin", + "admin@windmill.dev", + chrono::Utc::now(), + "admins", + benchmark_jobs + ) + .execute(db) + .await.unwrap_or_else(|_e| panic!("failed to insert noop jobs")); + } + } + } +} diff --git a/backend/windmill-common/src/ee.rs~main b/backend/windmill-common/src/ee.rs~main new file mode 100644 index 0000000000..482b61a0fe --- /dev/null +++ b/backend/windmill-common/src/ee.rs~main @@ -0,0 +1,73 @@ +#[cfg(feature = "enterprise")] +use crate::db::DB; +use crate::ee::LicensePlan::Community; +#[cfg(feature = "enterprise")] +use crate::error; +use serde::Deserialize; +use std::sync::Arc; +use tokio::sync::RwLock; + +lazy_static::lazy_static! { + pub static ref LICENSE_KEY_VALID: Arc> = Arc::new(RwLock::new(true)); + pub static ref LICENSE_KEY_ID: Arc> = Arc::new(RwLock::new("".to_string())); + pub static ref LICENSE_KEY: Arc> = Arc::new(RwLock::new("".to_string())); +} + +pub enum LicensePlan { + Community, + Pro, + Enterprise, +} + +pub async fn get_license_plan() -> LicensePlan { + // Implementation is not open source + return Community; +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum CriticalErrorChannel {} + +pub enum CriticalAlertKind { + #[cfg(feature = "enterprise")] + CriticalError, + #[cfg(feature = "enterprise")] + RecoveredCriticalError, +} + +#[cfg(feature = "enterprise")] +pub async fn send_critical_alert( + _error_message: String, + _db: &DB, + _kind: CriticalAlertKind, + _channels: Option>, +) { +} + +#[cfg(feature = "enterprise")] +pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () { + // Implementation is not open source +} + +#[cfg(feature = "enterprise")] +pub async fn renew_license_key( + _http_client: &reqwest::Client, + _db: &crate::db::DB, + _key: Option, + _manual: bool, +) -> String { + // Implementation is not open source + "".to_string() +} + +#[cfg(feature = "enterprise")] +pub async fn create_customer_portal_session( + _http_client: &reqwest::Client, + _key: Option, +) -> error::Result { + // Implementation is not open source + Ok("".to_string()) +} + +#[cfg(feature = "enterprise")] +pub async fn worker_groups_alerts(_db: &DB) {} diff --git a/backend/windmill-common/src/ee.rs~main_0 b/backend/windmill-common/src/ee.rs~main_0 new file mode 100644 index 0000000000..2f32756e45 --- /dev/null +++ b/backend/windmill-common/src/ee.rs~main_0 @@ -0,0 +1,76 @@ +#[cfg(feature = "enterprise")] +use crate::db::DB; +use crate::ee::LicensePlan::Community; +#[cfg(feature = "enterprise")] +use crate::error; +use serde::Deserialize; +use std::sync::Arc; +use tokio::sync::RwLock; + +lazy_static::lazy_static! { + pub static ref LICENSE_KEY_VALID: Arc> = Arc::new(RwLock::new(true)); + pub static ref LICENSE_KEY_ID: Arc> = Arc::new(RwLock::new("".to_string())); + pub static ref LICENSE_KEY: Arc> = Arc::new(RwLock::new("".to_string())); +} + +pub enum LicensePlan { + Community, + Pro, + Enterprise, +} + +pub async fn get_license_plan() -> LicensePlan { + // Implementation is not open source + return Community; +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum CriticalErrorChannel { + Email { email: String }, + Slack { slack_channel: String }, +} + +pub enum CriticalAlertKind { + #[cfg(feature = "enterprise")] + CriticalError, + #[cfg(feature = "enterprise")] + RecoveredCriticalError, +} + +#[cfg(feature = "enterprise")] +pub async fn send_critical_alert( + _error_message: String, + _db: &DB, + _kind: CriticalAlertKind, + _channels: Option>, +) { +} + +#[cfg(feature = "enterprise")] +pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () { + // Implementation is not open source +} + +#[cfg(feature = "enterprise")] +pub async fn renew_license_key( + _http_client: &reqwest::Client, + _db: &crate::db::DB, + _key: Option, + _manual: bool, +) -> String { + // Implementation is not open source + "".to_string() +} + +#[cfg(feature = "enterprise")] +pub async fn create_customer_portal_session( + _http_client: &reqwest::Client, + _key: Option, +) -> error::Result { + // Implementation is not open source + Ok("".to_string()) +} + +#[cfg(feature = "enterprise")] +pub async fn worker_groups_alerts(_db: &DB) {} diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index b1ad85da99..a5040a9da3 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -285,6 +285,23 @@ pub struct FlowModuleValueWithParallel { pub parallelism: Option, } +#[derive(Deserialize)] +pub struct FlowModuleValueWithSkipFailures { + pub skip_failures: Option, + pub parallel: Option, + pub parallelism: Option, +} + +#[derive(Deserialize)] +pub struct BranchWithSkipFailures { + pub skip_failure: Option, +} + +#[derive(Deserialize)] +pub struct FlowModuleWithBranches { + pub branches: Vec, +} + impl FlowModule { pub fn id_append(&mut self, s: &str) { self.id = format!("{}-{}", self.id, s); @@ -293,6 +310,16 @@ impl FlowModule { serde_json::from_str::(self.value.get()).map_err(crate::error::to_anyhow) } + pub fn get_value_with_skip_failures(&self) -> anyhow::Result { + serde_json::from_str::(self.value.get()) + .map_err(crate::error::to_anyhow) + } + + pub fn get_branches_skip_failures(&self) -> anyhow::Result { + serde_json::from_str::(self.value.get()) + .map_err(crate::error::to_anyhow) + } + pub fn is_flow(&self) -> bool { self.get_type().is_ok_and(|x| x == "flow") } diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 48c31fbebb..cd8059251e 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -17,6 +17,8 @@ use scripts::ScriptLang; use sqlx::{Pool, Postgres}; pub mod apps; +#[cfg(feature = "benchmark")] +pub mod bench; pub mod db; pub mod ee; pub mod error; @@ -51,6 +53,17 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; +#[macro_export] +macro_rules! add_time { + ($bench:expr, $name:expr) => { + #[cfg(feature = "benchmark")] + { + $bench.add_timing($name); + // println!("{}: {:?}", $z, $y.elapsed()); + } + }; +} + lazy_static::lazy_static! { pub static ref METRICS_PORT: u16 = std::env::var("METRICS_PORT") .ok() diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 8bbe541f07..8676878352 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -38,7 +38,7 @@ lazy_static::lazy_static! { .build().unwrap(); } -#[derive(Deserialize)] +#[derive(Deserialize, Clone)] pub struct Pagination { pub page: Option, pub per_page: Option, @@ -82,6 +82,12 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) { (per_page, offset) } +pub fn paginate_without_limits(pagination: Pagination) -> (usize, usize) { + let per_page = pagination.per_page.unwrap_or(MAX_PER_PAGE); + let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page; + (per_page, offset) +} + pub async fn now_from_db<'c, E: sqlx::PgExecutor<'c>>( db: E, ) -> Result> { diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index ae89cb7c6c..435b7c71f7 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -12,7 +12,7 @@ path = "src/lib.rs" default = [] enterprise = ["windmill-common/enterprise"] cloud = [] -benchmark = [] +benchmark = ["windmill-common/benchmark"] prometheus = ["dep:prometheus"] [dependencies] diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index ec2e688871..eca874aa63 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -37,9 +37,9 @@ use ulid::Ulid; use uuid::Uuid; use windmill_audit::audit_ee::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; -#[cfg(not(feature = "enterprise"))] -use windmill_common::worker::PriorityTags; + use windmill_common::{ + add_time, auth::{fetch_authed_from_permissioned_as, permissioned_as_to_username}, db::{Authed, UserDB}, error::{self, to_anyhow, Error}, @@ -62,9 +62,12 @@ use windmill_common::{ to_raw_value, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, NO_LOGS, WORKER_CONFIG, WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY, }, - BASE_URL, DB, METRICS_ENABLED, + DB, METRICS_ENABLED, }; +#[cfg(feature = "enterprise")] +use windmill_common::BASE_URL; + #[cfg(feature = "cloud")] use windmill_common::users::SUPERADMIN_SYNC_EMAIL; @@ -125,10 +128,12 @@ const MAX_FREE_CONCURRENT_RUNS: i32 = 30; const ERROR_HANDLER_USERNAME: &str = "error_handler"; const SCHEDULE_ERROR_HANDLER_USERNAME: &str = "schedule_error_handler"; +#[cfg(feature = "enterprise")] const SCHEDULE_RECOVERY_HANDLER_USERNAME: &str = "schedule_recovery_handler"; const ERROR_HANDLER_USER_GROUP: &str = "g/error_handler"; const ERROR_HANDLER_USER_EMAIL: &str = "error_handler@windmill.dev"; const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill.dev"; +#[cfg(feature = "enterprise")] const SCHEDULE_RECOVERY_HANDLER_USER_EMAIL: &str = "schedule_recovery_handler@windmill.dev"; #[derive(Clone, Debug)] @@ -174,6 +179,8 @@ pub async fn cancel_single_job<'c>( rsmq.clone(), "server", false, + #[cfg(feature = "benchmark")] + &mut windmill_common::bench::BenchmarkIter::new(), ) .await; @@ -477,6 +484,7 @@ where #[derive(Deserialize)] struct RawFlowFailureModule { + #[cfg(feature = "enterprise")] failure_module: Option>, } @@ -490,6 +498,7 @@ pub async fn add_completed_job_error, _worker_name: &str, flow_is_done: bool, + #[cfg(feature = "benchmark")] bench: &mut windmill_common::bench::BenchmarkIter, ) -> Result { #[cfg(feature = "prometheus")] register_metric( @@ -527,6 +536,8 @@ pub async fn add_completed_job_error, rsmq: Option, flow_is_done: bool, + #[cfg(feature = "benchmark")] bench: &mut windmill_common::bench::BenchmarkIter, ) -> Result { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); + add_time!(bench, "add_completed_job start"); if !result.is_valid_json() { return Err(Error::InternalErr( "Result of job is invalid json (empty)".to_string(), @@ -561,6 +574,7 @@ pub async fn add_completed_job< } let mut tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into(); + let job_id = queued_job.id; // tracing::error!("1 {:?}", start.elapsed()); @@ -571,6 +585,7 @@ pub async fn add_completed_job< ); let mem_peak = mem_peak.max(queued_job.mem_peak.unwrap_or(0)); + add_time!(bench, "add_completed_job query START"); let _duration: i64 = sqlx::query_scalar!( "INSERT INTO completed_job AS cj ( workspace_id @@ -642,6 +657,8 @@ pub async fn add_completed_job< .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e:#}")))?; // tracing::error!("2 {:?}", start.elapsed()); + add_time!(bench, "add_completed_job query END"); + if !queued_job.is_flow_step { if _duration > 500 && (queued_job.job_kind == JobKind::Script || queued_job.job_kind == JobKind::Preview) @@ -671,6 +688,7 @@ pub async fn add_completed_job< } } // tracing::error!("Added completed job {:#?}", queued_job); + #[cfg(feature = "enterprise")] let mut skip_downstream_error_handlers = false; tx = delete_job(tx, &queued_job.workspace_id, job_id).await?; // tracing::error!("3 {:?}", start.elapsed()); @@ -716,7 +734,10 @@ pub async fn add_completed_job< .await?; if let Some(schedule) = schedule { - skip_downstream_error_handlers = schedule.ws_error_handler_muted; + #[cfg(feature = "enterprise")] + { + skip_downstream_error_handlers = schedule.ws_error_handler_muted; + } // script or flow that failed on start and might not have been rescheduled let schedule_next_tick = !queued_job.is_flow() @@ -749,6 +770,7 @@ pub async fn add_completed_job< }; } + #[cfg(feature = "enterprise")] if let Err(err) = apply_schedule_handlers( rsmq.clone(), db, @@ -1324,6 +1346,8 @@ struct CompletedJobSubset { result: Option>>, started_at: chrono::DateTime, } + +#[cfg(feature = "enterprise")] async fn apply_schedule_handlers< 'a, 'c, @@ -1342,7 +1366,6 @@ async fn apply_schedule_handlers< job_priority: Option, ) -> windmill_common::error::Result<()> { if !success { - #[cfg(feature = "enterprise")] if let Some(on_failure_path) = schedule.on_failure.clone() { let times = schedule.on_failure_times.unwrap_or(1).max(1); let exact = schedule.on_failure_exact.unwrap_or(false); @@ -1392,7 +1415,6 @@ async fn apply_schedule_handlers< .await?; } } else { - #[cfg(feature = "enterprise")] if let Some(ref on_success_path) = schedule.on_success { handle_successful_schedule( db, @@ -1410,7 +1432,6 @@ async fn apply_schedule_handlers< .await?; } - #[cfg(feature = "enterprise")] if let Some(ref on_recovery_path) = schedule.on_recovery.clone() { let tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into(); let times = schedule.on_recovery_times.unwrap_or(1).max(1); @@ -1579,6 +1600,7 @@ fn sanitize_result(result: Json<&T>) -> HashMap( db: &Pool, rsmq: Option, suspend_first: bool, -) -> windmill_common::error::Result> { +) -> windmill_common::error::Result<(Option, bool)> { loop { - let job = pull_single_job_and_mark_as_running_no_concurrency_limit( + let (job, suspended) = pull_single_job_and_mark_as_running_no_concurrency_limit( db, rsmq.clone(), suspend_first, @@ -1765,7 +1788,7 @@ pub async fn pull( .await?; if job.is_none() { - return Ok(None); + return Ok((None, suspended)); } let has_concurent_limit = job.as_ref().unwrap().concurrent_limit.is_some(); @@ -1785,7 +1808,7 @@ pub async fn pull( if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { QUEUE_PULL_COUNT.inc(); } - return Ok(Option::Some(pulled_job)); + return Ok((Option::Some(pulled_job), suspended)); } let itx = db.begin().await?; @@ -1886,7 +1909,7 @@ pub async fn pull( QUEUE_PULL_COUNT.inc(); } tx.commit().await?; - return Ok(Option::Some(pulled_job)); + return Ok((Option::Some(pulled_job), suspended)); } let x = sqlx::query_scalar!( "UPDATE concurrency_counter SET job_uuids = job_uuids - $2 WHERE concurrency_id = $1 RETURNING (SELECT COUNT(*) FROM jsonb_object_keys(job_uuids))", @@ -2013,8 +2036,8 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit< db: &Pool, rsmq: Option, suspend_first: bool, -) -> windmill_common::error::Result> { - let job: Option = if let Some(mut rsmq) = rsmq { +) -> windmill_common::error::Result<(Option, bool)> { + let job_and_suspended: (Option, bool) = if let Some(mut rsmq) = rsmq { #[cfg(feature = "benchmark")] let instant = Instant::now(); @@ -2072,9 +2095,9 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit< #[cfg(feature = "benchmark")] println!("rsmq 2: {:?}", instant.elapsed()); - m2r + (m2r, false) } else { - None + (None, false) } } else { /* Jobs can be started if they: @@ -2088,7 +2111,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit< if query.is_empty() { tracing::warn!("No suspended pull queries available"); - return Ok(None); + return Ok((None, false)); } let r = if suspend_first { @@ -2108,7 +2131,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit< if queries.is_empty() { tracing::warn!("No pull queries available"); - return Ok(None); + return Ok((None, false)); } for query in queries.iter() { @@ -2126,12 +2149,12 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit< // #[cfg(feature = "benchmark")] // println!("pull query: {:?}", instant.elapsed()); - highest_priority_job + (highest_priority_job, false) } else { - r + (r, true) } }; - Ok(job) + Ok(job_and_suspended) } pub async fn custom_concurrency_key( diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 1529866492..c05bc26fc3 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -12,7 +12,7 @@ path = "src/lib.rs" default = [] prometheus = ["dep:prometheus", "windmill-common/prometheus"] enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:gcp_auth", "dep:pem", "dep:tiberius", "dep:tokio-util", "dep:openidconnect"] -benchmark = ["windmill-queue/benchmark"] +benchmark = ["windmill-queue/benchmark", "windmill-common/benchmark"] flamegraph = [] parquet = ["windmill-common/parquet", "dep:object_store"] flow_testing = [] diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index 17d5fcce4e..c718f0ec30 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -80,6 +80,12 @@ mount { rw: true } +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + exec_bin { path: "/bin/sh" arg: "/download_deps.sh" diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index 9cea96e4b7..efd6274832 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -19,6 +19,6 @@ then echo "\$TRUSTED_HOST is set to $TRUSTED_HOST" fi -CMD="/usr/local/bin/python3 -m pip install -v $REQ -I -t $TARGET --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG" +CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG" echo $CMD eval $CMD diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 6368c034e0..9fbb90ecc1 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -120,6 +120,7 @@ mount { src: "{JOB_DIR}/args.json" dst: "/tmp/{LANG}/args.json" is_bind: true + rw: true } mount { diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index 27b3677e54..bbdf5ed306 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -81,6 +81,7 @@ mount { src: "{JOB_DIR}/args.json" dst: "/tmp/args.json" is_bind: true + rw: true } mount { diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 297458633a..6639189f61 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -21,9 +21,10 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ bash_executor::BIN_BASH, common::{ - get_reserved_variables, handle_child, read_and_check_result, start_child_process, - transform_json, + get_reserved_variables, read_and_check_result, start_child_process, transform_json, + OccupancyMetrics, }, + handle_child::handle_child, python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV, @@ -50,6 +51,7 @@ async fn handle_ansible_python_deps( worker_dir: &str, mem_peak: &mut i32, canceled_by: &mut Option, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { create_dependencies_dir(job_dir).await; @@ -79,6 +81,7 @@ async fn handle_ansible_python_deps( db, worker_name, w_id, + &mut Some(occupancy_metrics), ) .await .map_err(|e| { @@ -102,6 +105,7 @@ async fn handle_ansible_python_deps( worker_name, job_dir, worker_dir, + &mut Some(occupancy_metrics), ) .await?; additional_python_paths.append(&mut venv_path); @@ -118,6 +122,7 @@ async fn install_galaxy_collections( mem_peak: &mut i32, canceled_by: &mut Option, db: &sqlx::Pool, + occupancy_metrics: &mut OccupancyMetrics, ) -> anyhow::Result<()> { write_file(job_dir, "requirements.yml", collections_yml)?; @@ -160,6 +165,7 @@ async fn install_galaxy_collections( "ansible galaxy install", None, false, + &mut Some(occupancy_metrics), ) .await?; @@ -198,6 +204,7 @@ pub async fn handle_ansible_job( shared_mount: &str, base_internal_url: &str, envs: HashMap, + occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { check_ansible_exists()?; @@ -216,6 +223,7 @@ pub async fn handle_ansible_job( worker_dir, mem_peak, canceled_by, + occupancy_metrics, ) .await?; @@ -289,6 +297,7 @@ pub async fn handle_ansible_job( mem_peak, canceled_by, db, + occupancy_metrics, ) .await?; } @@ -424,6 +433,7 @@ fi "python run", job.timeout, false, + &mut Some(occupancy_metrics), ) .await?; read_and_check_result(job_dir).await diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 412eb6b440..e9b135d277 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -24,9 +24,10 @@ lazy_static::lazy_static! { use crate::{ common::{ - build_args_map, get_reserved_variables, handle_child, read_file, read_file_content, - start_child_process, + build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process, + OccupancyMetrics, }, + handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV, }; @@ -49,6 +50,7 @@ pub async fn handle_bash_job( base_internal_url: &str, worker_name: &str, envs: HashMap, + occupancy_metrics: &mut OccupancyMetrics, ) -> Result, Error> { let logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string(); append_logs(&job.id, &job.workspace_id, logs1, db).await; @@ -142,6 +144,7 @@ pub async fn handle_bash_job( "bash run", job.timeout, true, + &mut Some(occupancy_metrics), ) .await?; @@ -194,6 +197,7 @@ pub async fn handle_powershell_job( base_internal_url: &str, worker_name: &str, envs: HashMap, + occupancy_metrics: &mut OccupancyMetrics, ) -> Result, Error> { let pwsh_args = { let args = build_args_map(job, client, db).await?.map(Json); @@ -275,6 +279,7 @@ pub async fn handle_powershell_job( "powershell install", job.timeout, false, + &mut Some(occupancy_metrics), ) .await?; } @@ -378,6 +383,7 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", "powershell run", job.timeout, false, + &mut Some(occupancy_metrics), ) .await?; diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index ea240b87c9..1552686b79 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -14,7 +14,8 @@ use windmill_queue::{CanceledBy, HTTP_CLIENT}; use serde::Deserialize; -use crate::common::run_future_with_polling_update_job_poller; +use crate::common::OccupancyMetrics; +use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{ common::{build_args_values, resolve_job_timeout}, AuthedClientBackgroundTask, @@ -210,6 +211,7 @@ pub async fn do_bigquery( canceled_by: &mut Option, worker_name: &str, column_order: &mut Option>, + occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { let bigquery_args = build_args_values(job, client, db).await?; @@ -364,6 +366,7 @@ pub async fn do_bigquery( result_f.map_err(to_anyhow), worker_name, &job.workspace_id, + &mut Some(occupancy_metrics), ) .await?; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index ca2f00d4b0..e17f140b32 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -13,10 +13,11 @@ use crate::common::build_envs_map; use crate::{ common::{ - create_args_and_out_file, get_main_override, get_reserved_variables, handle_child, - parse_npm_config, read_file, read_file_content, read_result, start_child_process, - write_file_binary, + create_args_and_out_file, get_main_override, get_reserved_variables, parse_npm_config, + read_file, read_file_content, read_result, start_child_process, write_file_binary, + OccupancyMetrics, }, + handle_child::handle_child, AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV, @@ -69,6 +70,7 @@ pub async fn gen_bun_lockfile( export_pkg: bool, raw_deps: Option, npm_mode: bool, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> Result> { let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(None).await; @@ -125,6 +127,7 @@ pub async fn gen_bun_lockfile( "bun build", None, false, + occupancy_metrics, ) .await?; } else { @@ -149,6 +152,7 @@ pub async fn gen_bun_lockfile( worker_name, common_bun_proc_envs, npm_mode, + occupancy_metrics, ) .await?; } else { @@ -230,6 +234,7 @@ pub async fn install_bun_lockfile( worker_name: &str, common_bun_proc_envs: HashMap, npm_mode: bool, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> Result<()> { let mut child_cmd = Command::new(if npm_mode { &*NPM_PATH } else { &*BUN_PATH }); child_cmd @@ -296,6 +301,7 @@ pub async fn install_bun_lockfile( "bun install", None, false, + occupancy_metrics, ) .await? } else { @@ -436,6 +442,7 @@ pub async fn generate_wrapper_mjs( mem_peak: &mut i32, canceled_by: &mut Option, common_bun_proc_envs: &HashMap, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> Result<()> { let mut child = Command::new(&*BUN_PATH); child @@ -459,6 +466,7 @@ pub async fn generate_wrapper_mjs( "bun build", timeout, false, + occupancy_metrics, ) .await?; fs::rename( @@ -479,6 +487,7 @@ pub async fn generate_bun_bundle( mem_peak: &mut i32, canceled_by: &mut Option, common_bun_proc_envs: &HashMap, + occupancy_metrics: &mut OccupancyMetrics, ) -> Result<()> { let mut child = Command::new(&*BUN_PATH); child @@ -503,6 +512,7 @@ pub async fn generate_bun_bundle( "bun build", timeout, false, + &mut Some(occupancy_metrics), ) .await?; } else { @@ -609,6 +619,7 @@ pub async fn prebundle_bun_script( base_internal_url: &str, worker_name: &str, token: &str, + occupancy_metrics: &mut OccupancyMetrics, ) -> Result<()> { let (local_path, remote_path) = compute_bundle_local_and_remote_path( inner_content, @@ -656,6 +667,7 @@ pub async fn prebundle_bun_script( &mut 0, &mut None, &common_bun_proc_envs, + occupancy_metrics, ) .await?; @@ -751,6 +763,7 @@ pub async fn handle_bun_job( envs: HashMap, shared_mount: &str, new_args: &mut Option>>, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { let mut annotation = windmill_common::worker::get_annotation(inner_content); @@ -861,6 +874,7 @@ pub async fn handle_bun_job( worker_name, common_bun_proc_envs.clone(), annotation.npm_mode, + &mut Some(occupancy_metrics), ) .await?; @@ -888,7 +902,6 @@ pub async fn handle_bun_job( // if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry { let logs1 = "\n\n--- BUN INSTALL ---\n".to_string(); append_logs(&job.id, &job.workspace_id, logs1, db).await; - let _ = gen_bun_lockfile( mem_peak, canceled_by, @@ -903,6 +916,7 @@ pub async fn handle_bun_job( false, None, annotation.npm_mode, + &mut Some(occupancy_metrics), ) .await?; @@ -1128,6 +1142,7 @@ try {{ mem_peak, canceled_by, &common_bun_proc_envs, + occupancy_metrics, ) .await?; if !local_path.is_empty() { @@ -1169,6 +1184,7 @@ try {{ mem_peak, canceled_by, &common_bun_proc_envs, + &mut Some(occupancy_metrics), ) .await?; } @@ -1204,6 +1220,7 @@ try {{ worker_name, &job.workspace_id, false, + occupancy_metrics, ) .await?; tracing::info!( @@ -1360,6 +1377,7 @@ try {{ "bun run", job.timeout, false, + &mut Some(occupancy_metrics), ) .await?; @@ -1501,6 +1519,7 @@ pub async fn start_worker( worker_name, common_bun_proc_envs.clone(), annotation.npm_mode, + &mut None, ) .await?; tracing::info!("dedicated worker requirements installed: {reqs}"); @@ -1521,6 +1540,7 @@ pub async fn start_worker( false, None, annotation.npm_mode, + &mut None, ) .await?; } @@ -1617,6 +1637,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) { &mut mem_peak, &mut canceled_by, &common_bun_proc_envs, + &mut None, ) .await?; } diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 1cda8fb03b..406f128917 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1,15 +1,7 @@ use async_recursion::async_recursion; -use deno_ast::swc::parser::lexer::util::CharExt; -use futures::Future; + use itertools::Itertools; -#[cfg(any(target_os = "linux", target_os = "macos"))] -use nix::sys::signal::{self, Signal}; -#[cfg(any(target_os = "linux", target_os = "macos"))] -use nix::unistd::Pid; - -#[cfg(all(feature = "enterprise", feature = "parquet"))] -use object_store::path::Path; use regex::Regex; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; @@ -18,18 +10,15 @@ use sqlx::types::Json; use sqlx::{Pool, Postgres}; use tokio::process::Command; use tokio::{fs::File, io::AsyncReadExt}; -use windmill_common::error::to_anyhow; use windmill_common::jobs::ENTRYPOINT_OVERRIDE; -#[cfg(all(feature = "enterprise", feature = "parquet"))] -use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS; + #[cfg(feature = "parquet")] use windmill_common::s3_helpers::{ get_etag_or_empty, LargeFileStorage, ObjectStoreResource, S3Object, }; use windmill_common::variables::{build_crypt_with_key_suffix, decrypt_value_with_mc}; use windmill_common::worker::{ - get_windmill_memory_usage, get_worker_memory_usage, to_raw_value, write_file, CLOUD_HOSTED, - ROOT_CACHE_DIR, TMP_DIR, WORKER_CONFIG, + to_raw_value, write_file, CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG, }; use windmill_common::{ error::{self, Error}, @@ -38,43 +27,21 @@ use windmill_common::{ }; use anyhow::{anyhow, Result}; -use windmill_queue::{append_logs, CanceledBy}; -#[cfg(any(target_os = "linux", target_os = "macos"))] -use std::os::unix::process::ExitStatusExt; - -use std::process::ExitStatus; -use std::sync::atomic::AtomicU32; -use std::sync::Arc; use std::{ collections::{hash_map::DefaultHasher, HashMap}, hash::{Hash, Hasher}, - io, panic, time::Duration, }; -use tracing::{trace_span, Instrument}; use uuid::Uuid; use windmill_common::{variables, DB}; -#[cfg(feature = "enterprise")] -use windmill_common::job_metrics; - -use tokio::{ - io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, - process::Child, - sync::{broadcast, watch}, - time::{interval, sleep, Instant, MissedTickBehavior}, -}; - -use futures::{ - future::{self, ready, FutureExt}, - stream, StreamExt, -}; +use tokio::{io::AsyncWriteExt, process::Child, time::Instant}; use crate::{ AuthedClient, AuthedClientBackgroundTask, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, - MAX_TIMEOUT_DURATION, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM, + MAX_TIMEOUT_DURATION, }; pub async fn build_args_map<'a>( @@ -466,36 +433,6 @@ pub fn get_main_override(args: Option<&Json>>>) -> .flatten(); } -async fn get_mem_peak(pid: Option, nsjail: bool) -> i32 { - if pid.is_none() { - return -1; - } - let pid = if nsjail { - // This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1. - // Ideally, we would get the number from fork() itself. This works in MOST cases. - pid.unwrap() + 1 - } else { - pid.unwrap() - }; - - if let Ok(file) = File::open(format!("/proc/{}/status", pid)).await { - let mut lines = BufReader::new(file).lines(); - while let Some(line) = lines.next_line().await.unwrap_or(None) { - if line.starts_with("VmHWM:") { - return line - .split_whitespace() - .nth(1) - .and_then(|s| s.parse::().ok()) - .unwrap_or(-1); - }; - } - -2 - } else { - // rand::random::() % 100 // to remove - used to fake memory data on MacOS - -3 - } -} - pub fn sizeof_val(v: &serde_json::Value) -> usize { std::mem::size_of::() + match v { @@ -516,63 +453,6 @@ pub fn sizeof_val(v: &serde_json::Value) -> usize { } } -pub async fn run_future_with_polling_update_job_poller( - job_id: Uuid, - timeout: Option, - db: &DB, - mem_peak: &mut i32, - canceled_by_ref: &mut Option, - result_f: Fut, - worker_name: &str, - w_id: &str, -) -> error::Result -where - Fut: Future>, -{ - let (tx, rx) = broadcast::channel::<()>(3); - - let update_job = update_job_poller( - job_id, - db, - mem_peak, - canceled_by_ref, - || async { 0 }, - worker_name, - w_id, - rx, - ); - - let timeout_ms = u64::try_from( - resolve_job_timeout(&db, &w_id, job_id, timeout) - .await - .0 - .as_millis(), - ) - .unwrap_or(200000); - - let rows = tokio::select! { - biased; - result = tokio::time::timeout(std::time::Duration::from_millis(timeout_ms), result_f) => result - .map_err(|e| { - tracing::error!("Query timeout: {}", e); - Error::ExecutionErr(format!("Query timeout after (>{}s)", timeout_ms/1000)) - })?, - ex = update_job, if job_id != Uuid::nil() => { - match ex { - UpdateJobPollingExit::Done => Err(Error::ExecutionErr("Job cancelled".to_string())).map_err(to_anyhow)?, - UpdateJobPollingExit::AlreadyCompleted => Err(Error::AlreadyCompleted("Job already completed".to_string())).map_err(to_anyhow)?, - } - } - }?; - drop(tx); - Ok(rows) -} - -pub enum UpdateJobPollingExit { - Done, - AlreadyCompleted, -} - pub async fn update_worker_ping_for_failed_init_script( db: &DB, worker_name: &str, @@ -594,672 +474,83 @@ pub async fn update_worker_ping_for_failed_init_script( tracing::error!("Error updating worker ping for failed init script: {e:?}"); } } +pub struct OccupancyMetrics { + pub running_job_started_at: Option, + pub total_duration_of_running_jobs: f32, + pub worker_occupancy_rate_history: Vec<(f32, f32)>, + pub start_time: Instant, +} -pub async fn update_job_poller( - job_id: Uuid, - db: &DB, - mem_peak: &mut i32, - canceled_by_ref: &mut Option, - get_mem: F, - worker_name: &str, - w_id: &str, - mut rx: broadcast::Receiver<()>, -) -> UpdateJobPollingExit -where - F: Fn() -> Fut, - Fut: Future, -{ - let update_job_interval = Duration::from_millis(500); +impl OccupancyMetrics { + pub fn new(start_time: Instant) -> Self { + OccupancyMetrics { + running_job_started_at: None, + total_duration_of_running_jobs: 0.0, + worker_occupancy_rate_history: Vec::new(), + start_time, + } + } - let db = db.clone(); + pub fn update_occupancy_metrics(&mut self) -> (f32, Option, Option, Option) { + let metrics = self; + let current_occupied_duration = metrics + .running_job_started_at + .map(|started_at| started_at.elapsed().as_secs_f32()) + .unwrap_or(0.0); + let total_occupation = metrics.total_duration_of_running_jobs + current_occupied_duration; - let mut interval = interval(update_job_interval); - interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + let elapsed = metrics.start_time.elapsed().as_secs_f32(); - let mut i = 0; - - #[cfg(feature = "enterprise")] - let mut memory_metric_id: Result = - Err(Error::NotFound("not yet initialized".to_string())); - - loop { - tokio::select!( - _ = rx.recv() => break, - _ = interval.tick() => { - // update the last_ping column every 5 seconds - i+=1; - if i == 1 || i % 10 == 0 { - let memory_usage = get_worker_memory_usage(); - let wm_memory_usage = get_windmill_memory_usage(); - tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024); - if job_id != Uuid::nil() { - sqlx::query!( - "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4 WHERE worker = $5", - &job_id, - &w_id, - memory_usage, - wm_memory_usage, - &worker_name - ) - .execute(&db) - .await - .expect("update worker ping"); - } - } - let current_mem = get_mem().await; - if current_mem > *mem_peak { - *mem_peak = current_mem - } - tracing::info!("job {job_id} on {worker_name} in {w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB"); - - - let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0; - if update_job_row { - #[cfg(feature = "enterprise")] + let (occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m) = + if !metrics.worker_occupancy_rate_history.is_empty() { + let mut total_occupation_15s = 0.0; + let mut total_occupation_5m = 0.0; + let mut total_occupation_30m = 0.0; + let mut index30m = 0; + for (i, (past_total_occupation, time)) in + metrics.worker_occupancy_rate_history.iter().enumerate() { - if job_id != Uuid::nil() { - - // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs - if i == 2 { - memory_metric_id = job_metrics::register_metric_for_job( - &db, - w_id.to_string(), - job_id, - "memory_kb".to_string(), - job_metrics::MetricKind::TimeseriesInt, - Some("Job Memory Footprint (kB)".to_string()), - ) - .await; - } - if let Ok(ref metric_id) = memory_metric_id { - if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await { - tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); - } - } + let diff = elapsed - time; + if diff < 1800.0 && total_occupation_30m == 0.0 { + total_occupation_30m = (total_occupation - past_total_occupation) / diff; + index30m = i; + } + if diff < 300.0 && total_occupation_5m == 0.0 { + total_occupation_5m = (total_occupation - past_total_occupation) / diff; + } + if diff < 15.0 { + total_occupation_15s = (total_occupation - past_total_occupation) / diff; + break; } } - if job_id != Uuid::nil() { - let (canceled, canceled_by, canceled_reason, already_completed) = sqlx::query_as::<_, (bool, Option, Option, bool)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason, false") - .bind(*mem_peak) - .bind(job_id) - .fetch_optional(&db) - .await - .unwrap_or_else(|e| { - tracing::error!(%e, "error updating job {job_id}: {e:#}"); - Some((false, None, None, false)) - }) - .unwrap_or_else(|| { - // if the job is not in queue, it can only be in the completed_job so it is already complete - (false, None, None, true) - }); - if already_completed { - return UpdateJobPollingExit::AlreadyCompleted - } - if canceled { - canceled_by_ref.replace(CanceledBy { - username: canceled_by.clone(), - reason: canceled_reason.clone(), - }); - break - } - } - } - }, - ); - } - tracing::info!("job {job_id} finished"); - UpdateJobPollingExit::Done -} + //drop all elements before the oldest one in 30m windows + metrics.worker_occupancy_rate_history.drain(..index30m); -pub enum CompactLogs { - NotEE, - NoS3, - S3, -} - -async fn compact_logs( - job_id: Uuid, - w_id: &str, - db: &DB, - nlogs: String, - total_size: Arc, - compact_kind: CompactLogs, - _worker_name: &str, -) -> error::Result<(String, String)> { - let mut prev_logs = sqlx::query_scalar!( - "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2", - job_id, - w_id - ) - .fetch_optional(db) - .await? - .flatten() - .unwrap_or_default(); - let size = prev_logs.char_indices().count() as i32; - let nlogs_len = nlogs.char_indices().count(); - let to_keep_in_db = usize::max( - usize::min(nlogs_len, 3000), - nlogs_len % LARGE_LOG_THRESHOLD_SIZE, - ); - let extra_split = to_keep_in_db < nlogs_len; - let stored_in_storage_len = if extra_split { - nlogs_len - to_keep_in_db - } else { - 0 - }; - let extra_to_newline = nlogs - .chars() - .skip(stored_in_storage_len) - .find_position(|x| x.is_line_break()) - .map(|(i, _)| i) - .unwrap_or(to_keep_in_db); - let stored_in_storage_to_newline = stored_in_storage_len + extra_to_newline; - - let (append_to_storage, stored_in_db) = if extra_split { - if stored_in_storage_to_newline == nlogs.len() { - (nlogs.as_ref(), "".to_string()) - } else { - let split_idx = nlogs - .char_indices() - .nth(stored_in_storage_to_newline) - .map(|(i, _)| i) - .unwrap_or(0); - let (append_to_storage, stored_in_db) = nlogs.split_at(split_idx); - // tracing::error!("{append_to_storage} ||||| {stored_in_db}"); - // tracing::error!( - // "{:?} {:?} {} {}", - // excess_prev_logs.lines().last(), - // current_logs.lines().next(), - // split_idx, - // excess_size_modulo - // ); - (append_to_storage, stored_in_db.to_string()) - } - } else { - // tracing::error!("{:?}", nlogs.lines().last()); - ("", nlogs.to_string()) - }; - - let new_size_with_excess = size + stored_in_storage_to_newline as i32; - - let new_size = total_size.fetch_add( - new_size_with_excess as u32, - std::sync::atomic::Ordering::SeqCst, - ) + new_size_with_excess as u32; - - let path = format!( - "logs/{job_id}/{}_{new_size}.txt", - chrono::Utc::now().timestamp_millis() - ); - - let mut new_current_logs = match compact_kind { - CompactLogs::NoS3 => format!("\n[windmill] No object storage set in instance settings. Previous logs have been saved to disk at {path}"), - CompactLogs::S3 => format!("\n[windmill] Previous logs have been saved to object storage at {path}"), - CompactLogs::NotEE => format!("\n[windmill] Previous logs have been saved to disk at {path}"), - }; - new_current_logs.push_str(&stored_in_db); - - sqlx::query!( - "UPDATE job_logs SET logs = $1, log_offset = $2, - log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3) - WHERE workspace_id = $4 AND job_id = $5", - new_current_logs, - new_size as i32, - path, - w_id, - job_id - ) - .execute(db) - .await?; - prev_logs.push_str(&append_to_storage); - - return Ok((prev_logs, path)); -} - -async fn default_disk_log_storage( - job_id: Uuid, - w_id: &str, - db: &DB, - nlogs: String, - total_size: Arc, - compact_kind: CompactLogs, - worker_name: &str, -) { - match compact_logs( - job_id, - &w_id, - &db, - nlogs, - total_size, - compact_kind, - worker_name, - ) - .await - { - Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",), - Ok((prev_logs, path)) => { - let path = format!("{}/{}", TMP_DIR, path); - let splitted = &path.split("/").collect_vec(); - tokio::fs::create_dir_all(splitted.into_iter().take(splitted.len() - 1).join("/")) - .await - .map_err(|e| { - tracing::error!("Could not create logs directory: {e:?}",); - e - }) - .ok(); - let created = tokio::fs::File::create(&path).await; - if let Err(e) = created { - tracing::error!("Could not create logs file {path}: {e:?}",); - return; - } - if let Err(e) = tokio::fs::write(&path, prev_logs).await { - tracing::error!("Could not write to logs file {path}: {e:?}"); - } else { - tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to disk at {path}"); - } - } - } -} - -async fn append_job_logs( - job_id: Uuid, - w_id: String, - logs: String, - db: DB, - must_compact_logs: bool, - total_size: Arc, - worker_name: String, -) -> () { - if must_compact_logs { - #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { - match compact_logs( - job_id, - &w_id, - &db, - logs, - total_size, - CompactLogs::S3, - &worker_name, - ) - .await - { - Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",), - Ok((prev_logs, path)) => { - tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to object storage at {path}"); - let path2 = path.clone(); - if let Err(e) = os - .put(&Path::from(path), prev_logs.to_string().into_bytes().into()) - .await - { - tracing::error!("Could not save logs to s3: {e:?}"); - } - tracing::info!("Logs of {job_id} saved to object storage at {path2}"); - } - } - } else { - default_disk_log_storage( - job_id, - &w_id, - &db, - logs, - total_size, - CompactLogs::NoS3, - &worker_name, - ) - .await; - } - - #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - { - default_disk_log_storage( - job_id, - &w_id, - &db, - logs, - total_size, - CompactLogs::NotEE, - &worker_name, - ) - .await; - } - } else { - append_logs(&job_id, w_id, logs, db).await; - } -} - -pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000; -/// - wait until child exits and return with exit status -/// - read lines from stdout and stderr and append them to the "queue"."logs" -/// quitting early if output exceedes MAX_LOG_SIZE characters (not bytes) -/// - update the `last_line` and `logs` strings with the program output -/// - update "queue"."last_ping" every five seconds -/// - kill process if we exceed timeout or "queue"."canceled" is set -#[tracing::instrument(level = "trace", skip_all)] -pub async fn handle_child( - job_id: &Uuid, - db: &Pool, - mem_peak: &mut i32, - canceled_by_ref: &mut Option, - mut child: Child, - nsjail: bool, - worker: &str, - w_id: &str, - child_name: &str, - custom_timeout: Option, - sigterm: bool, -) -> error::Result<()> { - let start = Instant::now(); - - let pid = child.id(); - #[cfg(target_os = "linux")] - if let Some(pid) = pid { - //set the highest oom priority - if let Some(mut file) = File::create(format!("/proc/{pid}/oom_score_adj")) - .await - .map_err(|e| { - tracing::error!("Could not create oom_score_file to pid {pid}: {e:#}"); - e - }) - .ok() - { - let _ = file.write_all(b"1000").await; - let _ = file.sync_all().await; - } - } else { - tracing::info!("could not get child pid"); - } - let (set_too_many_logs, mut too_many_logs) = watch::channel::(false); - let (tx, rx) = broadcast::channel::<()>(3); - let mut rx2 = tx.subscribe(); - - let output = child_joined_output_stream(&mut child); - - let job_id = job_id.clone(); - - /* the cancellation future is polled on by `wait_on_child` while - * waiting for the child to exit normally */ - let update_job = update_job_poller( - job_id, - db, - mem_peak, - canceled_by_ref, - || get_mem_peak(pid, nsjail), - worker, - w_id, - rx, - ); - - #[derive(PartialEq, Debug)] - enum KillReason { - TooManyLogs, - Timeout, - Cancelled, - AlreadyCompleted, - } - - let (timeout_duration, timeout_warn_msg) = - resolve_job_timeout(&db, w_id, job_id, custom_timeout).await; - if let Some(msg) = timeout_warn_msg { - append_logs(&job_id, w_id, msg.as_str(), db).await; - } - - /* a future that completes when the child process exits */ - let wait_on_child = async { - let db = db.clone(); - - let kill_reason = tokio::select! { - biased; - result = child.wait() => return result.map(Ok), - Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs, - _ = sleep(timeout_duration) => KillReason::Timeout, - ex = update_job, if job_id != Uuid::nil() => match ex { - UpdateJobPollingExit::Done => KillReason::Cancelled, - UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted, - }, - }; - tx.send(()).expect("rx should never be dropped"); - drop(tx); - - let set_reason = async { - if kill_reason == KillReason::Timeout { - if let Err(err) = sqlx::query( - r#" - UPDATE queue - SET canceled = true - , canceled_by = 'timeout' - , canceled_reason = $1 - WHERE id = $2 - "#, + ( + Some(total_occupation_15s), + Some(total_occupation_5m), + Some(total_occupation_30m), ) - .bind(format!("duration > {}", timeout_duration.as_secs())) - .bind(job_id) - .execute(&db) - .await - { - tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}"); - } - } - }; - - if let Some(id) = child.id() { - if *MAX_WAIT_FOR_SIGINT > 0 { - #[cfg(any(target_os = "linux", target_os = "macos"))] - signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap(); - - for _ in 0..*MAX_WAIT_FOR_SIGINT { - if child.try_wait().is_ok_and(|x| x.is_some()) { - break; - } - sleep(Duration::from_secs(1)).await; - } - if child.try_wait().is_ok_and(|x| x.is_some()) { - set_reason.await; - return Ok(Err(kill_reason)); - } - } - if sigterm { - #[cfg(any(target_os = "linux", target_os = "macos"))] - signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap(); - - for _ in 0..*MAX_WAIT_FOR_SIGTERM { - if child.try_wait().is_ok_and(|x| x.is_some()) { - break; - } - sleep(Duration::from_secs(1)).await; - } - if child.try_wait().is_ok_and(|x| x.is_some()) { - set_reason.await; - return Ok(Err(kill_reason)); - } - } - } - /* send SIGKILL and reap child process */ - let (_, kill) = future::join(set_reason, child.kill()).await; - kill.map(|()| Err(kill_reason)) - }; - - /* a future that reads output from the child and appends to the database */ - let lines = async move { - - let max_log_size = if *CLOUD_HOSTED { - MAX_RESULT_SIZE - } else { - usize::MAX - }; - - /* log_remaining is zero when output limit was reached */ - let mut log_remaining = if *CLOUD_HOSTED { - max_log_size - } else { - usize::MAX - }; - let mut result = io::Result::Ok(()); - let mut output = output.take_until(async { - let _ = rx2.recv().await; - //wait at most 50ms after end of a script for output stream to end - tokio::time::sleep(Duration::from_millis(50)).await; - }).boxed(); - /* `do_write` resolves the task, but does not contain the Result. - * It's useful to know if the task completed. */ - let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle(); - - let mut log_total_size: u64 = 0; - let pg_log_total_size = Arc::new(AtomicU32::new(0)); - - while let Some(line) = output.by_ref().next().await { - - let do_write_ = do_write.shared(); - - let delay = if start.elapsed() < Duration::from_secs(10) { - Duration::from_millis(500) - } else if start.elapsed() < Duration::from_secs(60){ - Duration::from_millis(2500) } else { - Duration::from_millis(5000) + (None, None, None) }; + let occupancy_rate = total_occupation / elapsed; - let delay = if *SLOW_LOGS { - delay * 10 - } else { - delay - }; + //push the current occupancy rate and the timestamp + metrics + .worker_occupancy_rate_history + .push((total_occupation, elapsed)); - let mut read_lines = stream::once(async { line }) - .chain(output.by_ref()) - /* after receiving a line, continue until some delay has passed - * _and_ the previous database write is complete */ - .take_until(future::join(sleep(delay), do_write_.clone())) - .boxed(); - - /* Read up until an error is encountered, - * handle log lines first and then the error... */ - let mut joined = String::new(); - - while let Some(line) = read_lines.next().await { - - match line { - Ok(line) => { - if line.is_empty() { - continue; - } - append_with_limit(&mut joined, &line, &mut log_remaining); - if log_remaining == 0 { - tracing::info!(%job_id, "Too many logs lines for job {job_id}"); - let _ = set_too_many_logs.send(true); - joined.push_str(&format!( - "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job." - )); - /* stop reading and drop our streams fairly quickly */ - break; - } - } - Err(err) => { - result = Err(err); - break; - } - } - } - - - /* Ensure the last flush completed before starting a new one. - * - * This shouldn't pause since `take_until()` reads lines until `do_write` - * resolves. We only stop reading lines before `take_until()` resolves if we reach - * EOF or a read error. In those cases, waiting on a database query to complete is - * fine because we're done. */ - - if let Some(Ok(p)) = do_write_ - .then(|()| write_result) - .await - .err() - .map(|err| err.try_into_panic()) - { - panic::resume_unwind(p); - } - - - let joined_len = joined.len() as u64; - log_total_size += joined_len; - let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64; - if compact_logs { - log_total_size = 0; - } - - let worker_name = worker.to_string(); - let w_id2 = w_id.to_string(); - (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle(); - - - - if let Err(err) = result { - tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}"); - break; - } - - if *set_too_many_logs.borrow() { - break; - } - } - - /* drop our end of the pipe */ - drop(output); - - if let Some(Ok(p)) = do_write - .then(|()| write_result) - .await - .err() - .map(|err| err.try_into_panic()) - { - panic::resume_unwind(p); - } - }.instrument(trace_span!("child_lines")); - - let (wait_result, _) = tokio::join!(wait_on_child, lines); - - let success = wait_result.is_ok() - && wait_result.as_ref().unwrap().is_ok() - && wait_result.as_ref().unwrap().as_ref().unwrap().success(); - tracing::info!(%job_id, %success, %mem_peak, %worker, "child process '{child_name}' took {}ms", start.elapsed().as_millis()); - - match wait_result { - _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!( - "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)" - ))), - Ok(Ok(status)) => process_status(status), - Ok(Err(kill_reason)) => match kill_reason { - KillReason::AlreadyCompleted => { - Err(Error::AlreadyCompleted("Job already completed".to_string())) - } - _ => Err(Error::ExecutionErr(format!( - "job process killed because {kill_reason:#?}" - ))), - }, - Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))), + ( + occupancy_rate, + occupancy_rate_15s, + occupancy_rate_5m, + occupancy_rate_30m, + ) } } -pub fn process_status(status: ExitStatus) -> error::Result<()> { - if status.success() { - Ok(()) - } else if let Some(code) = status.code() { - Err(error::Error::ExitStatus(code)) - } else { - #[cfg(any(target_os = "linux", target_os = "macos"))] - return Err(error::Error::ExecutionErr(format!( - "process terminated by signal: {:#?}, stopped_signal: {:#?}, core_dumped: {}", - status.signal(), - status.stopped_signal(), - status.core_dumped() - ))); - - #[cfg(not(any(target_os = "linux", target_os = "macos")))] - return Err(error::Error::ExecutionErr(String::from( - "process terminated by signal", - ))); - } -} pub async fn start_child_process(mut cmd: Command, executable: &str) -> Result { return cmd .spawn() @@ -1324,80 +615,6 @@ pub async fn resolve_job_timeout( } } -/// takes stdout and stderr from Child, panics if either are not present -/// -/// builds a stream joining both stdout and stderr each read line by line -fn child_joined_output_stream( - child: &mut Child, -) -> impl stream::FusedStream> { - let stderr = child - .stderr - .take() - .expect("child did not have a handle to stdout"); - - let stdout = child - .stdout - .take() - .expect("child did not have a handle to stdout"); - - let stdout = BufReader::new(stdout).lines(); - let stderr = BufReader::new(stderr).lines(); - stream::select(lines_to_stream(stderr), lines_to_stream(stdout)) -} - -pub fn lines_to_stream( - mut lines: tokio::io::Lines, -) -> impl futures::Stream> { - stream::poll_fn(move |cx| { - std::pin::Pin::new(&mut lines) - .poll_next_line(cx) - .map(|result| result.transpose()) - }) -} - -lazy_static::lazy_static! { - static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap(); - pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true"); - pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); -} -// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields, -// so this pushes \n to thd destination string in each call -fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) { - if *NO_LOGS_AT_ALL { - return; - } - let src_str; - let src = { - src_str = RE_00.replace_all(src, ""); - src_str.as_ref() - }; - if !*CLOUD_HOSTED { - dst.push('\n'); - dst.push_str(&src); - return; - } else { - if *limit > 0 { - dst.push('\n'); - } - *limit -= 1; - } - - let src_len = src.chars().count(); - if src_len <= *limit { - dst.push_str(&src); - *limit -= src_len; - } else { - let byte_pos = src - .char_indices() - .skip(*limit) - .next() - .map(|(byte_pos, _)| byte_pos) - .unwrap_or(0); - dst.push_str(&src[0..byte_pos]); - *limit = 0; - } -} - pub async fn hash_args( _db: &DB, _client: &AuthedClient, diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index fbbc161de9..da3e25a699 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -29,8 +29,7 @@ use windmill_queue::append_logs; use anyhow::Context; use crate::{ - common::{process_status, start_child_process}, - JobCompleted, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS, + common::start_child_process, JobCompleted, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS, }; use futures::{future, Future}; @@ -78,6 +77,8 @@ pub async fn handle_dedicated_process( ) -> std::result::Result<(), error::Error> { //do not cache local dependencies + use crate::handle_child::process_status; + let mut child = { let mut cmd = Command::new(command_path); cmd.current_dir(job_dir) @@ -127,7 +128,7 @@ pub async fn handle_dedicated_process( if let Err(e) = process_status(status) { tracing::error!("child exit status was not success: {e:#}"); } else { - tracing::info!("child exist status was success"); + tracing::info!("child exit status was success"); } }); diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 146d67be07..fbb875a012 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -7,9 +7,10 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - create_args_and_out_file, get_main_override, get_reserved_variables, handle_child, - parse_npm_config, read_file, read_result, start_child_process, + create_args_and_out_file, get_main_override, get_reserved_variables, parse_npm_config, + read_file, read_result, start_child_process, OccupancyMetrics, }, + handle_child::handle_child, AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; @@ -94,6 +95,7 @@ pub async fn generate_deno_lock( w_id: &str, worker_name: &str, base_internal_url: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result { let _ = write_file(job_dir, "main.ts", code)?; @@ -146,6 +148,7 @@ pub async fn generate_deno_lock( "deno cache", None, false, + occupancy_metrics, ) .await?; } else { @@ -173,6 +176,7 @@ pub async fn handle_deno_job( worker_name: &str, envs: HashMap, new_args: &mut Option>>, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { // let mut start = Instant::now(); let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); @@ -397,6 +401,7 @@ try {{ "deno run", job.timeout, false, + &mut Some(occupancy_metrics), ) .await?; // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str()); diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 202584c653..b139859005 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -15,9 +15,10 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - capitalize, create_args_and_out_file, get_reserved_variables, handle_child, read_result, - start_child_process, + capitalize, create_args_and_out_file, get_reserved_variables, read_result, + start_child_process, OccupancyMetrics, }, + handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV, }; @@ -44,6 +45,7 @@ pub async fn handle_go_job( base_internal_url: &str, worker_name: &str, envs: HashMap, + occupation_metrics: &mut OccupancyMetrics, ) -> Result, Error> { //go does not like executing modules at temp root let job_dir = &format!("{job_dir}/go"); @@ -88,6 +90,7 @@ pub async fn handle_go_job( skip_tidy, worker_name, &job.workspace_id, + occupation_metrics, ) .await?; @@ -202,6 +205,7 @@ func Run(req Req) (interface{{}}, error){{ "go build", None, false, + &mut Some(occupation_metrics), ) .await?; @@ -297,6 +301,7 @@ func Run(req Req) (interface{{}}, error){{ "go run", job.timeout, false, + &mut Some(occupation_metrics), ) .await?; @@ -336,6 +341,7 @@ pub async fn install_go_dependencies( has_sum: bool, worker_name: &str, w_id: &str, + occupation_metrics: &mut OccupancyMetrics, ) -> error::Result { if !skip_go_mod { gen_go_mymod(code, job_dir).await?; @@ -359,6 +365,7 @@ pub async fn install_go_dependencies( "go init", None, false, + &mut Some(occupation_metrics), ) .await?; @@ -424,9 +431,9 @@ pub async fn install_go_dependencies( &format!("go {mod_command}"), None, false, + &mut Some(occupation_metrics), ) - .await - .map_err(|e| Error::ExecutionErr(format!("Lockfile generation failed: {e:?}")))?; + .await?; if (!new_lockfile || has_sum) && non_dep_job { return Ok("".to_string()); diff --git a/backend/windmill-worker/src/graphql_executor.rs b/backend/windmill-worker/src/graphql_executor.rs index db1d47920f..8fb8cf13b9 100644 --- a/backend/windmill-worker/src/graphql_executor.rs +++ b/backend/windmill-worker/src/graphql_executor.rs @@ -12,7 +12,8 @@ use windmill_queue::{CanceledBy, HTTP_CLIENT}; use serde::Deserialize; -use crate::common::run_future_with_polling_update_job_poller; +use crate::common::OccupancyMetrics; +use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{common::build_args_map, AuthedClientBackgroundTask}; #[derive(Deserialize)] @@ -41,6 +42,7 @@ pub async fn do_graphql( mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, + occupation_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { let args = build_args_map(job, client, db).await?.map(Json); let job_args = if args.is_some() { @@ -151,6 +153,7 @@ pub async fn do_graphql( result_f, worker_name, &job.workspace_id, + &mut Some(occupation_metrics), ) .await?; diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs new file mode 100644 index 0000000000..d588ea8fc4 --- /dev/null +++ b/backend/windmill-worker/src/handle_child.rs @@ -0,0 +1,627 @@ +use futures::Future; + +#[cfg(any(target_os = "linux", target_os = "macos"))] +use nix::sys::signal::{self, Signal}; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use nix::unistd::Pid; + +use sqlx::{Pool, Postgres}; +use tokio::fs::File; +use windmill_common::error::to_anyhow; + +use windmill_common::error::{self, Error}; + +use windmill_common::worker::{get_windmill_memory_usage, get_worker_memory_usage, CLOUD_HOSTED}; + +use windmill_queue::{append_logs, CanceledBy}; + +#[cfg(any(target_os = "linux", target_os = "macos"))] +use std::os::unix::process::ExitStatusExt; + +use std::process::ExitStatus; +use std::sync::atomic::AtomicU32; +use std::sync::Arc; +use std::{io, panic, time::Duration}; + +use tracing::{trace_span, Instrument}; +use uuid::Uuid; +use windmill_common::DB; + +#[cfg(feature = "enterprise")] +use windmill_common::job_metrics; + +use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + process::Child, + sync::{broadcast, watch}, + time::{interval, sleep, Instant, MissedTickBehavior}, +}; + +use futures::{ + future::{self, ready, FutureExt}, + stream, StreamExt, +}; + +use crate::common::{resolve_job_timeout, OccupancyMetrics}; +use crate::job_logger::{append_job_logs, append_with_limit, LARGE_LOG_THRESHOLD_SIZE}; +use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM}; + +lazy_static::lazy_static! { + pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); +} +/// - wait until child exits and return with exit status +/// - read lines from stdout and stderr and append them to the "queue"."logs" +/// quitting early if output exceedes MAX_LOG_SIZE characters (not bytes) +/// - update the `last_line` and `logs` strings with the program output +/// - update "queue"."last_ping" every five seconds +/// - kill process if we exceed timeout or "queue"."canceled" is set +#[tracing::instrument(level = "trace", skip_all)] +pub async fn handle_child( + job_id: &Uuid, + db: &Pool, + mem_peak: &mut i32, + canceled_by_ref: &mut Option, + mut child: Child, + nsjail: bool, + worker: &str, + w_id: &str, + child_name: &str, + custom_timeout: Option, + sigterm: bool, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, +) -> error::Result<()> { + let start = Instant::now(); + + let pid = child.id(); + #[cfg(target_os = "linux")] + if let Some(pid) = pid { + //set the highest oom priority + if let Some(mut file) = File::create(format!("/proc/{pid}/oom_score_adj")) + .await + .map_err(|e| { + tracing::error!("Could not create oom_score_file to pid {pid}: {e:#}"); + e + }) + .ok() + { + let _ = file.write_all(b"1000").await; + let _ = file.sync_all().await; + } + } else { + tracing::info!("could not get child pid"); + } + let (set_too_many_logs, mut too_many_logs) = watch::channel::(false); + let (tx, rx) = broadcast::channel::<()>(3); + let mut rx2 = tx.subscribe(); + + let output = child_joined_output_stream(&mut child); + + let job_id = job_id.clone(); + + /* the cancellation future is polled on by `wait_on_child` while + * waiting for the child to exit normally */ + let update_job = update_job_poller( + job_id, + db, + mem_peak, + canceled_by_ref, + || get_mem_peak(pid, nsjail), + worker, + w_id, + rx, + occupancy_metrics, + ); + + #[derive(PartialEq, Debug)] + enum KillReason { + TooManyLogs, + Timeout, + Cancelled, + AlreadyCompleted, + } + + let (timeout_duration, timeout_warn_msg) = + resolve_job_timeout(&db, w_id, job_id, custom_timeout).await; + if let Some(msg) = timeout_warn_msg { + append_logs(&job_id, w_id, msg.as_str(), db).await; + } + + /* a future that completes when the child process exits */ + let wait_on_child = async { + let db = db.clone(); + + let kill_reason = tokio::select! { + biased; + result = child.wait() => return result.map(Ok), + Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs, + _ = sleep(timeout_duration) => KillReason::Timeout, + ex = update_job, if job_id != Uuid::nil() => match ex { + UpdateJobPollingExit::Done => KillReason::Cancelled, + UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted, + }, + }; + tx.send(()).expect("rx should never be dropped"); + drop(tx); + + let set_reason = async { + if kill_reason == KillReason::Timeout { + if let Err(err) = sqlx::query( + r#" + UPDATE queue + SET canceled = true + , canceled_by = 'timeout' + , canceled_reason = $1 + WHERE id = $2 + "#, + ) + .bind(format!("duration > {}", timeout_duration.as_secs())) + .bind(job_id) + .execute(&db) + .await + { + tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}"); + } + } + }; + + if let Some(id) = child.id() { + if *MAX_WAIT_FOR_SIGINT > 0 { + #[cfg(any(target_os = "linux", target_os = "macos"))] + signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap(); + + for _ in 0..*MAX_WAIT_FOR_SIGINT { + if child.try_wait().is_ok_and(|x| x.is_some()) { + break; + } + sleep(Duration::from_secs(1)).await; + } + if child.try_wait().is_ok_and(|x| x.is_some()) { + set_reason.await; + return Ok(Err(kill_reason)); + } + } + if sigterm { + #[cfg(any(target_os = "linux", target_os = "macos"))] + signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap(); + + for _ in 0..*MAX_WAIT_FOR_SIGTERM { + if child.try_wait().is_ok_and(|x| x.is_some()) { + break; + } + sleep(Duration::from_secs(1)).await; + } + if child.try_wait().is_ok_and(|x| x.is_some()) { + set_reason.await; + return Ok(Err(kill_reason)); + } + } + } + /* send SIGKILL and reap child process */ + let (_, kill) = future::join(set_reason, child.kill()).await; + kill.map(|()| Err(kill_reason)) + }; + + /* a future that reads output from the child and appends to the database */ + let lines = async move { + + let max_log_size = if *CLOUD_HOSTED { + MAX_RESULT_SIZE + } else { + usize::MAX + }; + + /* log_remaining is zero when output limit was reached */ + let mut log_remaining = if *CLOUD_HOSTED { + max_log_size + } else { + usize::MAX + }; + let mut result = io::Result::Ok(()); + let mut output = output.take_until(async { + let _ = rx2.recv().await; + //wait at most 50ms after end of a script for output stream to end + tokio::time::sleep(Duration::from_millis(50)).await; + }).boxed(); + /* `do_write` resolves the task, but does not contain the Result. + * It's useful to know if the task completed. */ + let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle(); + + let mut log_total_size: u64 = 0; + let pg_log_total_size = Arc::new(AtomicU32::new(0)); + + while let Some(line) = output.by_ref().next().await { + + let do_write_ = do_write.shared(); + + let delay = if start.elapsed() < Duration::from_secs(10) { + Duration::from_millis(500) + } else if start.elapsed() < Duration::from_secs(60){ + Duration::from_millis(2500) + } else { + Duration::from_millis(5000) + }; + + let delay = if *SLOW_LOGS { + delay * 10 + } else { + delay + }; + + let mut read_lines = stream::once(async { line }) + .chain(output.by_ref()) + /* after receiving a line, continue until some delay has passed + * _and_ the previous database write is complete */ + .take_until(future::join(sleep(delay), do_write_.clone())) + .boxed(); + + /* Read up until an error is encountered, + * handle log lines first and then the error... */ + let mut joined = String::new(); + + while let Some(line) = read_lines.next().await { + + match line { + Ok(line) => { + if line.is_empty() { + continue; + } + append_with_limit(&mut joined, &line, &mut log_remaining); + if log_remaining == 0 { + tracing::info!(%job_id, "Too many logs lines for job {job_id}"); + let _ = set_too_many_logs.send(true); + joined.push_str(&format!( + "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job." + )); + /* stop reading and drop our streams fairly quickly */ + break; + } + } + Err(err) => { + result = Err(err); + break; + } + } + } + + + /* Ensure the last flush completed before starting a new one. + * + * This shouldn't pause since `take_until()` reads lines until `do_write` + * resolves. We only stop reading lines before `take_until()` resolves if we reach + * EOF or a read error. In those cases, waiting on a database query to complete is + * fine because we're done. */ + + if let Some(Ok(p)) = do_write_ + .then(|()| write_result) + .await + .err() + .map(|err| err.try_into_panic()) + { + panic::resume_unwind(p); + } + + + let joined_len = joined.len() as u64; + log_total_size += joined_len; + let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64; + if compact_logs { + log_total_size = 0; + } + + let worker_name = worker.to_string(); + let w_id2 = w_id.to_string(); + (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle(); + + + + if let Err(err) = result { + tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}"); + break; + } + + if *set_too_many_logs.borrow() { + break; + } + } + + /* drop our end of the pipe */ + drop(output); + + if let Some(Ok(p)) = do_write + .then(|()| write_result) + .await + .err() + .map(|err| err.try_into_panic()) + { + panic::resume_unwind(p); + } + }.instrument(trace_span!("child_lines")); + + let (wait_result, _) = tokio::join!(wait_on_child, lines); + + let success = wait_result.is_ok() + && wait_result.as_ref().unwrap().is_ok() + && wait_result.as_ref().unwrap().as_ref().unwrap().success(); + tracing::info!(%job_id, %success, %mem_peak, %worker, "child process '{child_name}' took {}ms", start.elapsed().as_millis()); + + match wait_result { + _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!( + "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)" + ))), + Ok(Ok(status)) => process_status(status), + Ok(Err(kill_reason)) => match kill_reason { + KillReason::AlreadyCompleted => { + Err(Error::AlreadyCompleted("Job already completed".to_string())) + } + _ => Err(Error::ExecutionErr(format!( + "job process killed because {kill_reason:#?}" + ))), + }, + Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))), + } +} + +async fn get_mem_peak(pid: Option, nsjail: bool) -> i32 { + if pid.is_none() { + return -1; + } + let pid = if nsjail { + // This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1. + // Ideally, we would get the number from fork() itself. This works in MOST cases. + pid.unwrap() + 1 + } else { + pid.unwrap() + }; + + if let Ok(file) = File::open(format!("/proc/{}/status", pid)).await { + let mut lines = BufReader::new(file).lines(); + while let Some(line) = lines.next_line().await.unwrap_or(None) { + if line.starts_with("VmHWM:") { + return line + .split_whitespace() + .nth(1) + .and_then(|s| s.parse::().ok()) + .unwrap_or(-1); + }; + } + -2 + } else { + // rand::random::() % 100 // to remove - used to fake memory data on MacOS + -3 + } +} + +pub async fn run_future_with_polling_update_job_poller( + job_id: Uuid, + timeout: Option, + db: &DB, + mem_peak: &mut i32, + canceled_by_ref: &mut Option, + result_f: Fut, + worker_name: &str, + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, +) -> error::Result +where + Fut: Future>, +{ + let (tx, rx) = broadcast::channel::<()>(3); + + let update_job = update_job_poller( + job_id, + db, + mem_peak, + canceled_by_ref, + || async { 0 }, + worker_name, + w_id, + rx, + occupancy_metrics, + ); + + let timeout_ms = u64::try_from( + resolve_job_timeout(&db, &w_id, job_id, timeout) + .await + .0 + .as_millis(), + ) + .unwrap_or(200000); + + let rows = tokio::select! { + biased; + result = tokio::time::timeout(std::time::Duration::from_millis(timeout_ms), result_f) => result + .map_err(|e| { + tracing::error!("Query timeout: {}", e); + Error::ExecutionErr(format!("Query timeout after (>{}s)", timeout_ms/1000)) + })?, + ex = update_job, if job_id != Uuid::nil() => { + match ex { + UpdateJobPollingExit::Done => Err(Error::ExecutionErr("Job cancelled".to_string())).map_err(to_anyhow)?, + UpdateJobPollingExit::AlreadyCompleted => Err(Error::AlreadyCompleted("Job already completed".to_string())).map_err(to_anyhow)?, + } + } + }?; + drop(tx); + Ok(rows) +} + +pub enum UpdateJobPollingExit { + Done, + AlreadyCompleted, +} + +pub async fn update_job_poller( + job_id: Uuid, + db: &DB, + mem_peak: &mut i32, + canceled_by_ref: &mut Option, + get_mem: F, + worker_name: &str, + w_id: &str, + mut rx: broadcast::Receiver<()>, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, +) -> UpdateJobPollingExit +where + F: Fn() -> Fut, + Fut: Future, +{ + let update_job_interval = Duration::from_millis(500); + + let db = db.clone(); + + let mut interval = interval(update_job_interval); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + let mut i = 0; + + #[cfg(feature = "enterprise")] + let mut memory_metric_id: Result = + Err(Error::NotFound("not yet initialized".to_string())); + + loop { + tokio::select!( + _ = rx.recv() => break, + _ = interval.tick() => { + // update the last_ping column every 5 seconds + i+=1; + if i == 1 || i % 10 == 0 { + let memory_usage = get_worker_memory_usage(); + let wm_memory_usage = get_windmill_memory_usage(); + tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024); + let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics()); + if job_id != Uuid::nil() { + sqlx::query!( + "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4, + occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5", + &job_id, + &w_id, + memory_usage, + wm_memory_usage, + &worker_name, + occupancy.map(|x| x.0), + occupancy.and_then(|x| x.1), + occupancy.and_then(|x| x.2), + occupancy.and_then(|x| x.3), + ) + .execute(&db) + .await + .expect("update worker ping"); + } + } + let current_mem = get_mem().await; + if current_mem > *mem_peak { + *mem_peak = current_mem + } + tracing::info!("job {job_id} on {worker_name} in {w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB"); + + + let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0; + if update_job_row { + #[cfg(feature = "enterprise")] + { + if job_id != Uuid::nil() { + + // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs + if i == 2 { + memory_metric_id = job_metrics::register_metric_for_job( + &db, + w_id.to_string(), + job_id, + "memory_kb".to_string(), + job_metrics::MetricKind::TimeseriesInt, + Some("Job Memory Footprint (kB)".to_string()), + ) + .await; + } + if let Ok(ref metric_id) = memory_metric_id { + if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await { + tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); + } + } + } + } + if job_id != Uuid::nil() { + let (canceled, canceled_by, canceled_reason, already_completed) = sqlx::query_as::<_, (bool, Option, Option, bool)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason, false") + .bind(*mem_peak) + .bind(job_id) + .fetch_optional(&db) + .await + .unwrap_or_else(|e| { + tracing::error!(%e, "error updating job {job_id}: {e:#}"); + Some((false, None, None, false)) + }) + .unwrap_or_else(|| { + // if the job is not in queue, it can only be in the completed_job so it is already complete + (false, None, None, true) + }); + if already_completed { + return UpdateJobPollingExit::AlreadyCompleted + } + if canceled { + canceled_by_ref.replace(CanceledBy { + username: canceled_by.clone(), + reason: canceled_reason.clone(), + }); + break + } + } + } + }, + ); + } + tracing::info!("job {job_id} finished"); + + UpdateJobPollingExit::Done +} + +/// takes stdout and stderr from Child, panics if either are not present +/// +/// builds a stream joining both stdout and stderr each read line by line +fn child_joined_output_stream( + child: &mut Child, +) -> impl stream::FusedStream> { + let stderr = child + .stderr + .take() + .expect("child did not have a handle to stdout"); + + let stdout = child + .stdout + .take() + .expect("child did not have a handle to stdout"); + + let stdout = BufReader::new(stdout).lines(); + let stderr = BufReader::new(stderr).lines(); + stream::select(lines_to_stream(stderr), lines_to_stream(stdout)) +} + +pub fn lines_to_stream( + mut lines: tokio::io::Lines, +) -> impl futures::Stream> { + stream::poll_fn(move |cx| { + std::pin::Pin::new(&mut lines) + .poll_next_line(cx) + .map(|result| result.transpose()) + }) +} + +pub fn process_status(status: ExitStatus) -> error::Result<()> { + if status.success() { + Ok(()) + } else if let Some(code) = status.code() { + Err(error::Error::ExitStatus(code)) + } else { + #[cfg(any(target_os = "linux", target_os = "macos"))] + return Err(error::Error::ExecutionErr(format!( + "process terminated by signal: {:#?}, stopped_signal: {:#?}, core_dumped: {}", + status.signal(), + status.stopped_signal(), + status.core_dumped() + ))); + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + return Err(error::Error::ExecutionErr(String::from( + "process terminated by signal", + ))); + } +} diff --git a/backend/windmill-worker/src/job_logger.rs b/backend/windmill-worker/src/job_logger.rs new file mode 100644 index 0000000000..5193fe79aa --- /dev/null +++ b/backend/windmill-worker/src/job_logger.rs @@ -0,0 +1,285 @@ +use deno_ast::swc::parser::lexer::util::CharExt; +use itertools::Itertools; + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +use object_store::path::Path; +use regex::Regex; + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS; + +use windmill_common::error::{self}; +use windmill_common::worker::{CLOUD_HOSTED, TMP_DIR}; + +use windmill_queue::append_logs; + +use std::sync::atomic::AtomicU32; +use std::sync::Arc; + +use uuid::Uuid; +use windmill_common::DB; + +pub enum CompactLogs { + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + NotEE, + #[allow(dead_code)] + NoS3, + #[allow(dead_code)] + S3, +} + +async fn compact_logs( + job_id: Uuid, + w_id: &str, + db: &DB, + nlogs: String, + total_size: Arc, + compact_kind: CompactLogs, + _worker_name: &str, +) -> error::Result<(String, String)> { + let mut prev_logs = sqlx::query_scalar!( + "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2", + job_id, + w_id + ) + .fetch_optional(db) + .await? + .flatten() + .unwrap_or_default(); + let size = prev_logs.char_indices().count() as i32; + let nlogs_len = nlogs.char_indices().count(); + let to_keep_in_db = usize::max( + usize::min(nlogs_len, 3000), + nlogs_len % LARGE_LOG_THRESHOLD_SIZE, + ); + let extra_split = to_keep_in_db < nlogs_len; + let stored_in_storage_len = if extra_split { + nlogs_len - to_keep_in_db + } else { + 0 + }; + let extra_to_newline = nlogs + .chars() + .skip(stored_in_storage_len) + .find_position(|x| x.is_line_break()) + .map(|(i, _)| i) + .unwrap_or(to_keep_in_db); + let stored_in_storage_to_newline = stored_in_storage_len + extra_to_newline; + + let (append_to_storage, stored_in_db) = if extra_split { + if stored_in_storage_to_newline == nlogs.len() { + (nlogs.as_ref(), "".to_string()) + } else { + let split_idx = nlogs + .char_indices() + .nth(stored_in_storage_to_newline) + .map(|(i, _)| i) + .unwrap_or(0); + let (append_to_storage, stored_in_db) = nlogs.split_at(split_idx); + // tracing::error!("{append_to_storage} ||||| {stored_in_db}"); + // tracing::error!( + // "{:?} {:?} {} {}", + // excess_prev_logs.lines().last(), + // current_logs.lines().next(), + // split_idx, + // excess_size_modulo + // ); + (append_to_storage, stored_in_db.to_string()) + } + } else { + // tracing::error!("{:?}", nlogs.lines().last()); + ("", nlogs.to_string()) + }; + + let new_size_with_excess = size + stored_in_storage_to_newline as i32; + + let new_size = total_size.fetch_add( + new_size_with_excess as u32, + std::sync::atomic::Ordering::SeqCst, + ) + new_size_with_excess as u32; + + let path = format!( + "logs/{job_id}/{}_{new_size}.txt", + chrono::Utc::now().timestamp_millis() + ); + + let mut new_current_logs = match compact_kind { + CompactLogs::NoS3 => format!("\n[windmill] No object storage set in instance settings. Previous logs have been saved to disk at {path}"), + CompactLogs::S3 => format!("\n[windmill] Previous logs have been saved to object storage at {path}"), + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + CompactLogs::NotEE => format!("\n[windmill] Previous logs have been saved to disk at {path}"), + }; + new_current_logs.push_str(&stored_in_db); + + sqlx::query!( + "UPDATE job_logs SET logs = $1, log_offset = $2, + log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3) + WHERE workspace_id = $4 AND job_id = $5", + new_current_logs, + new_size as i32, + path, + w_id, + job_id + ) + .execute(db) + .await?; + prev_logs.push_str(&append_to_storage); + + return Ok((prev_logs, path)); +} + +async fn default_disk_log_storage( + job_id: Uuid, + w_id: &str, + db: &DB, + nlogs: String, + total_size: Arc, + compact_kind: CompactLogs, + worker_name: &str, +) { + match compact_logs( + job_id, + &w_id, + &db, + nlogs, + total_size, + compact_kind, + worker_name, + ) + .await + { + Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",), + Ok((prev_logs, path)) => { + let path = format!("{}/{}", TMP_DIR, path); + let splitted = &path.split("/").collect_vec(); + tokio::fs::create_dir_all(splitted.into_iter().take(splitted.len() - 1).join("/")) + .await + .map_err(|e| { + tracing::error!("Could not create logs directory: {e:?}",); + e + }) + .ok(); + let created = tokio::fs::File::create(&path).await; + if let Err(e) = created { + tracing::error!("Could not create logs file {path}: {e:?}",); + return; + } + if let Err(e) = tokio::fs::write(&path, prev_logs).await { + tracing::error!("Could not write to logs file {path}: {e:?}"); + } else { + tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to disk at {path}"); + } + } + } +} + +pub(crate) async fn append_job_logs( + job_id: Uuid, + w_id: String, + logs: String, + db: DB, + must_compact_logs: bool, + total_size: Arc, + worker_name: String, +) -> () { + if must_compact_logs { + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { + match compact_logs( + job_id, + &w_id, + &db, + logs, + total_size, + CompactLogs::S3, + &worker_name, + ) + .await + { + Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",), + Ok((prev_logs, path)) => { + tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to object storage at {path}"); + let path2 = path.clone(); + if let Err(e) = os + .put(&Path::from(path), prev_logs.to_string().into_bytes().into()) + .await + { + tracing::error!("Could not save logs to s3: {e:?}"); + } + tracing::info!("Logs of {job_id} saved to object storage at {path2}"); + } + } + } else { + default_disk_log_storage( + job_id, + &w_id, + &db, + logs, + total_size, + CompactLogs::NoS3, + &worker_name, + ) + .await; + } + + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + { + default_disk_log_storage( + job_id, + &w_id, + &db, + logs, + total_size, + CompactLogs::NotEE, + &worker_name, + ) + .await; + } + } else { + append_logs(&job_id, w_id, logs, db).await; + } +} + +pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000; + +lazy_static::lazy_static! { + static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap(); + pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true"); +} +// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields, +// so this pushes \n to thd destination string in each call +pub fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) { + if *NO_LOGS_AT_ALL { + return; + } + let src_str; + let src = { + src_str = RE_00.replace_all(src, ""); + src_str.as_ref() + }; + if !*CLOUD_HOSTED { + dst.push('\n'); + dst.push_str(&src); + return; + } else { + if *limit > 0 { + dst.push('\n'); + } + *limit -= 1; + } + + let src_len = src.chars().count(); + if src_len <= *limit { + dst.push_str(&src); + *limit -= src_len; + } else { + let byte_pos = src + .char_indices() + .skip(*limit) + .next() + .map(|(byte_pos, _)| byte_pos) + .unwrap_or(0); + dst.push_str(&src[0..byte_pos]); + *limit = 0; + } +} diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index fb3a378861..24f83c86af 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -40,7 +40,8 @@ use windmill_common::{error::Error, flow_status::JobResult, DB}; use windmill_queue::CanceledBy; use crate::{ - common::{run_future_with_polling_update_job_poller, unsafe_raw}, + common::{unsafe_raw, OccupancyMetrics}, + handle_child::run_future_with_polling_update_job_poller, AuthedClient, }; @@ -688,6 +689,7 @@ pub async fn eval_fetch_timeout( worker_name: &str, w_id: &str, load_client: bool, + occupation_metrics: &mut OccupancyMetrics, ) -> anyhow::Result<(Box, String)> { let (sender, mut receiver) = oneshot::channel::(); @@ -836,6 +838,7 @@ pub async fn eval_fetch_timeout( async { result_f.await? }, worker_name, w_id, + &mut Some(occupation_metrics), ) .await .map_err(|e| { diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index cf5c601c90..f1c5bc3926 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -5,7 +5,9 @@ mod mssql_executor; #[cfg(feature = "enterprise")] mod snowflake_executor; +mod ansible_executor; mod bash_executor; + mod bun_executor; pub mod common; mod config; @@ -15,6 +17,8 @@ mod deno_executor; mod global_cache; mod go_executor; mod graphql_executor; +mod handle_child; +mod job_logger; mod js_eval; mod mysql_executor; mod pg_executor; @@ -25,7 +29,7 @@ mod rust_executor; mod worker; mod worker_flow; mod worker_lockfiles; -mod ansible_executor; + pub use worker::*; pub use result_processor::handle_job_error; diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index 2bffb2ee61..f50bad4619 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -14,7 +14,8 @@ use windmill_common::{error::to_anyhow, jobs::QueuedJob}; use windmill_parser_sql::{parse_db_resource, parse_mssql_sig}; use windmill_queue::{append_logs, CanceledBy}; -use crate::common::{build_args_values, run_future_with_polling_update_job_poller}; +use crate::common::{build_args_values, OccupancyMetrics}; +use crate::handle_child::run_future_with_polling_update_job_poller; use crate::AuthedClientBackgroundTask; #[derive(Deserialize)] @@ -39,6 +40,7 @@ pub async fn do_mssql( mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { let mssql_args = build_args_values(job, client, db).await?; @@ -157,6 +159,7 @@ pub async fn do_mssql( result_f, worker_name, &job.workspace_id, + &mut Some(occupancy_metrics), ) .await?; diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 9d16c6bde2..03cde1d508 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -22,7 +22,8 @@ use windmill_parser_sql::{ use windmill_queue::CanceledBy; use crate::{ - common::{build_args_map, run_future_with_polling_update_job_poller}, + common::{build_args_map, OccupancyMetrics}, + handle_child::run_future_with_polling_update_job_poller, AuthedClientBackgroundTask, }; @@ -110,6 +111,7 @@ pub async fn do_mysql( canceled_by: &mut Option, worker_name: &str, column_order: &mut Option>, + occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { let args = build_args_map(job, client, db).await?.map(Json); let job_args = if args.is_some() { @@ -295,6 +297,7 @@ pub async fn do_mysql( result_f, worker_name, &job.workspace_id, + &mut Some(occupancy_metrics), ) .await?; diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 000042d3e6..7c4192cf66 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -38,7 +38,8 @@ use windmill_parser_sql::{ }; use windmill_queue::CanceledBy; -use crate::common::{build_args_values, run_future_with_polling_update_job_poller, sizeof_val}; +use crate::common::{build_args_values, sizeof_val, OccupancyMetrics}; +use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{AuthedClientBackgroundTask, MAX_RESULT_SIZE}; use bytes::{Buf, BytesMut}; use lazy_static::lazy_static; @@ -162,6 +163,7 @@ pub async fn do_postgresql( canceled_by: &mut Option, worker_name: &str, column_order: &mut Option>, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { let pg_args = build_args_values(job, client, db).await?; @@ -351,6 +353,7 @@ pub async fn do_postgresql( result_f, worker_name, &job.workspace_id, + &mut Some(occupancy_metrics), ) .await?; diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index cd2f7e2499..e62d96b3fd 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -15,9 +15,10 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - create_args_and_out_file, get_main_override, get_reserved_variables, handle_child, - read_result, start_child_process, + create_args_and_out_file, get_main_override, get_reserved_variables, read_result, + start_child_process, OccupancyMetrics, }, + handle_child::handle_child, AuthedClientBackgroundTask, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, }; @@ -70,6 +71,7 @@ pub async fn composer_install( worker_name: &str, requirements: String, lock: Option, + occupancy_metrics: &mut OccupancyMetrics, ) -> Result { check_php_exists()?; @@ -101,6 +103,7 @@ pub async fn composer_install( "composer install", None, false, + &mut Some(occupancy_metrics), ) .await?; @@ -160,6 +163,7 @@ pub async fn handle_php_job( worker_name: &str, envs: HashMap, shared_mount: &str, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { check_php_exists()?; @@ -190,6 +194,7 @@ pub async fn handle_php_job( worker_name, composer_json, composer_lock, + occupancy_metrics, ) .await?; "require './vendor/autoload.php';" @@ -346,6 +351,7 @@ try {{ "php run", job.timeout, false, + &mut Some(occupancy_metrics), ) .await?; read_result(job_dir).await diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 51584eb037..87c1c829ae 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -55,9 +55,10 @@ use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS; use crate::{ common::{ - create_args_and_out_file, get_main_override, get_reserved_variables, handle_child, - read_file, read_result, start_child_process, + create_args_and_out_file, get_main_override, get_reserved_variables, read_file, + read_result, start_child_process, OccupancyMetrics, }, + handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, TZ_ENV, @@ -101,6 +102,7 @@ pub async fn pip_compile( db: &Pool, worker_name: &str, w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result { let mut logs = String::new(); logs.push_str(&format!("\nresolving dependencies...")); @@ -210,6 +212,7 @@ pub async fn pip_compile( "pip-compile", None, false, + occupancy_metrics, ) .await .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; @@ -247,6 +250,7 @@ pub async fn handle_python_job( base_internal_url: &str, envs: HashMap, new_args: &mut Option>>, + occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { let script_path = crate::common::use_flow_root_path(job.script_path()); let additional_python_paths = handle_python_deps( @@ -261,6 +265,7 @@ pub async fn handle_python_job( worker_dir, mem_peak, canceled_by, + &mut Some(occupancy_metrics), ) .await?; @@ -485,6 +490,7 @@ mount {{ "python run", job.timeout, false, + &mut Some(occupancy_metrics), ) .await?; @@ -753,6 +759,7 @@ async fn handle_python_deps( worker_dir: &str, mem_peak: &mut i32, canceled_by: &mut Option, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result> { create_dependencies_dir(job_dir).await; @@ -790,6 +797,7 @@ async fn handle_python_deps( db, worker_name, w_id, + occupancy_metrics, ) .await .map_err(|e| { @@ -813,6 +821,7 @@ async fn handle_python_deps( worker_name, job_dir, worker_dir, + occupancy_metrics, ) .await?; additional_python_paths.append(&mut venv_path); @@ -834,6 +843,7 @@ pub async fn handle_python_reqs( worker_name: &str, job_dir: &str, worker_dir: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result> { let mut req_paths: Vec = vec![]; let mut vars = vec![("PATH", PATH_ENV.as_str())]; @@ -1089,6 +1099,7 @@ pub async fn handle_python_reqs( &format!("pip install {req}"), None, false, + occupancy_metrics, ) .await; tracing::info!( @@ -1173,6 +1184,7 @@ pub async fn start_worker( job_dir, &mut mem_peak, &mut canceled_by, + &mut None, ) .await?; diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 5203a65432..6c4f43746f 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -1,16 +1,26 @@ use serde::Serialize; use sqlx::{types::Json, Pool, Postgres}; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicBool, AtomicU16, Ordering}, + Arc, + }, +}; use uuid::Uuid; use windmill_common::{ + add_time, error::{self, Error}, - jobs::QueuedJob, - worker::to_raw_value, + jobs::{JobKind, QueuedJob}, + worker::{to_raw_value, WORKER_GROUP}, DB, }; +#[cfg(feature = "benchmark")] +use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; + use windmill_queue::{append_logs, get_queued_job, CanceledBy, WrappedError}; #[cfg(feature = "prometheus")] @@ -18,7 +28,13 @@ use windmill_queue::register_metric; use serde_json::{json, value::RawValue}; -use tokio::sync::mpsc::Sender; +use tokio::{ + sync::{ + self, + mpsc::{Receiver, Sender}, + }, + task::JoinHandle, +}; use windmill_queue::{add_completed_job, add_completed_job_error}; @@ -26,9 +42,149 @@ use crate::{ bash_executor::ANSI_ESCAPE_RE, common::{read_result, save_in_cache}, worker_flow::update_flow_status_after_job_completion, - AuthedClient, Histo, JobCompleted, JobCompletedSender, SameWorkerSender, SendResult, + AuthedClient, JobCompleted, JobCompletedSender, SameWorkerSender, SendResult, INIT_SCRIPT_TAG, }; +pub fn start_background_processor( + mut job_completed_rx: Receiver, + job_completed_sender: Sender, + same_worker_queue_size: Arc, + job_completed_processor_is_done: Arc, + base_internal_url: String, + db: DB, + worker_dir: String, + same_worker_tx: SameWorkerSender, + rsmq: Option, + worker_name: String, + killpill_tx: sync::broadcast::Sender<()>, + is_dedicated_worker: bool, +) -> JoinHandle<()> +where + R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static, +{ + tokio::spawn(async move { + let mut has_been_killed = false; + + #[cfg(feature = "benchmark")] + let mut infos = BenchmarkInfo::new(); + + //if we have been killed, we want to drain the queue of jobs + while let Some(sr) = { + if has_been_killed && same_worker_queue_size.load(Ordering::SeqCst) == 0 { + job_completed_rx.try_recv().ok() + } else { + job_completed_rx.recv().await + } + } { + #[cfg(feature = "benchmark")] + let mut bench = BenchmarkIter::new(); + + match sr { + SendResult::JobCompleted(jc) => { + let rsmq = rsmq.clone(); + + let is_init_script_and_failure = + !jc.success && jc.job.tag.as_str() == INIT_SCRIPT_TAG; + let is_dependency_job = matches!( + jc.job.job_kind, + JobKind::Dependencies | JobKind::FlowDependencies + ); + + handle_receive_completed_job( + jc, + &base_internal_url, + &db, + &worker_dir, + &same_worker_tx, + rsmq, + &worker_name, + job_completed_sender.clone(), + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await; + + if is_init_script_and_failure { + tracing::error!("init script errored, exiting"); + killpill_tx.send(()).unwrap_or_default(); + break; + } + if is_dependency_job && is_dedicated_worker { + tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted."); + sqlx::query!( + "UPDATE config SET config = config WHERE name = $1", + format!("worker__{}", *WORKER_GROUP) + ) + .execute(&db) + .await + .expect("update config to trigger restart of all dedicated workers at that config"); + killpill_tx.send(()).unwrap_or_default(); + } + add_time!(bench, "job completed processed"); + + #[cfg(feature = "benchmark")] + { + infos.add_iter(bench, true); + } + } + SendResult::UpdateFlow { + flow, + w_id, + success, + result, + worker_dir, + stop_early_override, + token, + } => { + // let r; + tracing::info!(parent_flow = %flow, "updating flow status"); + if let Err(e) = update_flow_status_after_job_completion( + &db, + &AuthedClient { + base_internal_url: base_internal_url.to_string(), + workspace: w_id.clone(), + token: token.clone(), + force_client: None, + }, + flow, + &Uuid::nil(), + &w_id, + success, + Arc::new(result), + true, + same_worker_tx.clone(), + &worker_dir, + stop_early_override, + rsmq.clone(), + &worker_name, + job_completed_sender.clone(), + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await + { + tracing::error!("Error updating flow status after job completion for {flow} on {worker_name}: {e:#}"); + } + } + SendResult::Kill => { + has_been_killed = true; + } + } + } + + job_completed_processor_is_done.store(true, Ordering::SeqCst); + + tracing::info!("finished processing all completed jobs"); + + #[cfg(feature = "benchmark")] + { + infos + .write_to_file("profiling_result_processor.json") + .expect("write to file profiling"); + } + }) +} + async fn send_job_completed( job_completed_tx: JobCompletedSender, job: Arc, @@ -168,9 +324,8 @@ pub async fn handle_receive_completed_job< same_worker_tx: &SameWorkerSender, rsmq: Option, worker_name: &str, - worker_save_completed_job_duration: Option, - worker_flow_transition_duration: Option, job_completed_tx: Sender, + #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) { let token = jc.token.clone(); let workspace = jc.job.workspace_id.clone(); @@ -191,9 +346,9 @@ pub async fn handle_receive_completed_job< same_worker_tx.clone(), rsmq.clone(), worker_name, - worker_save_completed_job_duration, - worker_flow_transition_duration, job_completed_tx.clone(), + #[cfg(feature = "benchmark")] + bench, ) .await { @@ -210,6 +365,8 @@ pub async fn handle_receive_completed_job< rsmq.clone(), worker_name, job_completed_tx, + #[cfg(feature = "benchmark")] + bench, ) .await; } @@ -224,9 +381,8 @@ pub async fn process_completed_job, worker_name: &str, - _worker_save_completed_job_duration: Option, - _worker_flow_transition_duration: Option, job_completed_tx: Sender, + #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> windmill_common::error::Result<()> { if success { // println!("bef completed job{:?}", SystemTime::now()); @@ -238,10 +394,7 @@ pub async fn process_completed_job, worker_name: &str, job_completed_tx: Sender, + #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) { let err = match err { Error::JsonErr(err) => err, @@ -365,6 +521,8 @@ pub async fn handle_job_error, worker_name: &str, w_id: &str, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result { check_cargo_exists()?; @@ -137,6 +139,7 @@ pub async fn generate_cargo_lockfile( "cargo generate-lockfile", None, false, + &mut Some(occupancy_metrics), ) .await?; @@ -157,6 +160,7 @@ pub async fn build_rust_crate( w_id: &str, base_internal_url: &str, hash: &str, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result { let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); @@ -185,6 +189,7 @@ pub async fn build_rust_crate( "rust build", None, false, + &mut Some(occupancy_metrics), ) .await?; append_logs(job_id, w_id, "\n\n", db).await; @@ -262,6 +267,7 @@ pub async fn handle_rust_job( base_internal_url: &str, worker_name: &str, envs: HashMap, + occupancy_metrics: &mut OccupancyMetrics, ) -> Result, Error> { check_cargo_exists()?; @@ -305,6 +311,7 @@ pub async fn handle_rust_job( &job.workspace_id, base_internal_url, &hash, + occupancy_metrics, ) .await? }; @@ -367,6 +374,7 @@ pub async fn handle_rust_job( "rust run", job.timeout, false, + &mut Some(occupancy_metrics), ) .await?; read_result(job_dir).await diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 9c9bec7c9f..bf9dae41ff 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -18,7 +18,8 @@ use windmill_queue::{CanceledBy, HTTP_CLIENT}; use serde::{Deserialize, Serialize}; -use crate::common::{resolve_job_timeout, run_future_with_polling_update_job_poller}; +use crate::common::{resolve_job_timeout, OccupancyMetrics}; +use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{common::build_args_values, AuthedClientBackgroundTask}; #[derive(Serialize)] @@ -237,6 +238,7 @@ pub async fn do_snowflake( canceled_by: &mut Option, worker_name: &str, column_order: &mut Option>, + occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { let snowflake_args = build_args_values(job, client, db).await?; @@ -383,6 +385,7 @@ pub async fn do_snowflake( result_f.map_err(to_anyhow), worker_name, &job.workspace_id, + &mut Some(occupancy_metrics), ) .await?; *mem_peak = (r.get().len() / 1000) as i32; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 5c4a02431b..6c0fce3d3b 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -18,10 +18,8 @@ use windmill_common::{ use anyhow::{Context, Result}; use const_format::concatcp; #[cfg(feature = "prometheus")] -use prometheus::{ - core::{AtomicI64, GenericGauge}, - IntCounter, -}; +use prometheus::IntCounter; + use tracing::Instrument; #[cfg(feature = "prometheus")] use windmill_common::METRICS_DEBUG_ENABLED; @@ -32,14 +30,15 @@ use reqwest::Response; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sqlx::{types::Json, Pool, Postgres}; use std::{ - collections::{hash_map::DefaultHasher, HashMap}, fs::DirBuilder, hash::Hash, sync::{ + collections::{hash_map::DefaultHasher, HashMap}, + fs::DirBuilder, + hash::Hash, + sync::{ atomic::{AtomicBool, AtomicU16, Ordering}, Arc, - }, time::Duration + }, + time::Duration, }; -#[cfg(feature = "benchmark")] -use std::sync::atomic::AtomicUsize; - use uuid::Uuid; @@ -55,8 +54,8 @@ use windmill_common::{ }; use windmill_queue::{ - append_logs, canceled_job_to_result, empty_result, pull, push, CanceledBy, - PushArgs, PushIsolationLevel, HTTP_CLIENT, + append_logs, canceled_job_to_result, empty_result, pull, push, CanceledBy, PushArgs, + PushIsolationLevel, HTTP_CLIENT, }; #[cfg(feature = "prometheus")] @@ -81,15 +80,31 @@ use tokio::{ use rand::Rng; - use crate::{ - ansible_executor::handle_ansible_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{ - build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, update_worker_ping_for_failed_init_script, NO_LOGS_AT_ALL, SLOW_LOGS - }, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, handle_job_error, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{handle_receive_completed_job, process_result}, rust_executor::handle_rust_job, worker_flow::{ - handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, Step, - }, worker_lockfiles::{ + ansible_executor::handle_ansible_job, + bash_executor::{handle_bash_job, handle_powershell_job}, + bun_executor::handle_bun_job, + common::{ + build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, + update_worker_ping_for_failed_init_script, OccupancyMetrics, + }, + deno_executor::handle_deno_job, + go_executor::handle_go_job, + graphql_executor::do_graphql, + handle_child::SLOW_LOGS, + handle_job_error, + job_logger::NO_LOGS_AT_ALL, + js_eval::{eval_fetch_timeout, transpile_ts}, + mysql_executor::do_mysql, + pg_executor::do_postgresql, + php_executor::handle_php_job, + python_executor::handle_python_job, + result_processor::{process_result, start_background_processor}, + rust_executor::handle_rust_job, + worker_flow::{handle_flow, update_flow_status_in_progress, Step}, + worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, - } + }, }; #[cfg(feature = "enterprise")] @@ -100,6 +115,11 @@ use crate::{ bigquery_executor::do_bigquery, mssql_executor::do_mssql, snowflake_executor::do_snowflake, }; +#[cfg(feature = "benchmark")] +use windmill_common::bench::{benchmark_init, BenchmarkInfo, BenchmarkIter}; + +use windmill_common::add_time; + pub async fn create_token_for_owner_in_bg( db: &Pool, job: &QueuedJob, @@ -541,43 +561,13 @@ impl AuthedClient { } } -#[cfg(feature = "benchmark")] -#[derive(Serialize)] -struct BenchmarkInfo { - iters: u64, - timings: Vec>, -} - -#[macro_export] -macro_rules! add_time { - ($x:expr, $y:expr, $z:expr) => { - #[cfg(feature = "benchmark")] - { - $x.push($y.elapsed().as_nanos() as u32); - // println!("{}: {:?}", $z, $y.elapsed()); - } - }; -} - -#[cfg(feature = "prometheus")] -pub type Histo = Arc; -#[cfg(feature = "prometheus")] -type GGauge = Arc>; - -#[cfg(not(feature = "prometheus"))] -pub type Histo = (); -#[cfg(not(feature = "prometheus"))] -type GGauge = (); - #[allow(dead_code)] #[derive(Clone)] -pub struct JobCompletedSender(Sender, Option, Option); - +pub struct JobCompletedSender(Sender); #[derive(Clone)] pub struct SameWorkerSender(pub Sender, pub Arc); - pub struct SameWorkerPayload { pub job_id: Uuid, pub recoverable: bool, @@ -588,16 +578,7 @@ impl JobCompletedSender { &self, jc: JobCompleted, ) -> Result<(), tokio::sync::mpsc::error::SendError> { - #[cfg(feature = "prometheus")] - if let Some(wj) = self.1.as_ref() { - wj.inc() - } - #[cfg(feature = "prometheus")] - let timer = self.2.as_ref().map(|x| x.start_timer()); - let r = self.0.send(SendResult::JobCompleted(jc)).await; - #[cfg(feature = "prometheus")] - timer.map(|x| x.stop_and_record()); - r + self.0.send(SendResult::JobCompleted(jc)).await } } @@ -611,7 +592,6 @@ impl SameWorkerSender { } } - // on linux, we drop caches every DROP_CACHE_PERIOD to avoid OOM killer believing we are using too much memory just because we create lots of files when executing jobs #[cfg(any(target_os = "linux"))] pub async fn drop_cache() { @@ -802,45 +782,6 @@ pub async fn run_worker() + .unwrap(); + + #[cfg(feature = "benchmark")] + benchmark_init(benchmark_jobs, &db).await; + #[cfg(feature = "prometheus")] if let Some(ws) = WORKER_STARTED.as_ref() { ws.inc(); @@ -1039,218 +934,36 @@ pub async fn run_worker(5); - let (job_completed_tx, mut job_completed_rx) = mpsc::channel::(3); + let (job_completed_tx, job_completed_rx) = mpsc::channel::(3); - let job_completed_tx = JobCompletedSender( - job_completed_tx, - worker_job_completed_channel_queue.clone(), - worker_completed_channel_queue_send_duration, - ); + let job_completed_tx = JobCompletedSender(job_completed_tx); let same_worker_queue_size = Arc::new(AtomicU16::new(0)); let same_worker_tx = SameWorkerSender(same_worker_tx, same_worker_queue_size.clone()); - - let db2 = db.clone(); - let base_internal_url2 = base_internal_url.to_string(); - let same_worker_tx2 = same_worker_tx.clone(); - let rsmq2 = rsmq.clone(); - let worker_dir2 = worker_dir.clone(); - - let is_dedicated_worker = WORKER_CONFIG.read().await.dedicated_worker.is_some(); - - #[cfg(feature = "benchmark")] - let jobs = 25000; - - #[cfg(feature = "benchmark")] - { - if is_dedicated_worker { - // you need to create the script first, check https://github.com/windmill-labs/windmill/blob/b76a92cfe454c686f005c65f534e29e039f3c706/benchmarks/lib.ts#L47 - let hash = sqlx::query_scalar!( - "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2", - "f/benchmarks/dedicated", - "admins" - ) - .fetch_one(db) - .await - .unwrap_or_else(|_e| panic!("failed to insert dedicated jobs")); - sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))", - hash, - "f/benchmarks/dedicated", - JobKind::Script as JobKind, - ScriptLang::Bun as ScriptLang, - "admins:f/benchmarks/dedicated", - "admin", - "u/admin", - "admin@windmill.dev", - chrono::Utc::now(), - "admins", - jobs - ) - .execute(db) - .await.unwrap_or_else(|_e| panic!("failed to insert dedicated jobs")); - } else { - sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))", - None::, - None::, - JobKind::Noop as JobKind, - ScriptLang::Deno as ScriptLang, - "deno", - "admin", - "u/admin", - "admin@windmill.dev", - chrono::Utc::now(), - "admins", - jobs - ) - .execute(db) - .await.unwrap_or_else(|_e| panic!("failed to insert noop jobs")); - } - } - - #[cfg(feature = "benchmark")] - let completed_jobs = Arc::new(AtomicUsize::new(0)); - #[cfg(feature = "benchmark")] - let start = Instant::now(); - #[cfg(feature = "benchmark")] - let main_duration = Arc::new(AtomicUsize::new(0)); - #[cfg(feature = "benchmark")] - let send_duration = Arc::new(AtomicUsize::new(0)); - #[cfg(feature = "benchmark")] - let process_duration = Arc::new(AtomicUsize::new(0)); - - #[cfg(feature = "benchmark")] - let main_duration2 = main_duration.clone(); - #[cfg(feature = "benchmark")] - let send_duration2 = send_duration.clone(); - - #[cfg(feature = "prometheus")] - let worker_job_completed_channel_queue2 = worker_job_completed_channel_queue.clone(); - #[cfg(feature = "prometheus")] - let worker_save_completed_job_duration2 = worker_save_completed_job_duration.clone(); - #[cfg(feature = "prometheus")] - let worker_flow_transition_duration2 = worker_flow_transition_duration.clone(); - - #[cfg(not(feature = "prometheus"))] - let worker_save_completed_job_duration2 = None; - #[cfg(not(feature = "prometheus"))] - let worker_flow_transition_duration2 = None; - - let worker_name2 = worker_name.clone(); - let killpill_tx2 = killpill_tx.clone(); - let job_completed_sender = job_completed_tx.0.clone(); - let job_completed_processor_is_done = Arc::new(AtomicBool::new(false)); - let job_completed_processor_is_done2 = job_completed_processor_is_done.clone(); - let same_worker_queue_size2 = same_worker_queue_size.clone(); - - - let send_result = tokio::spawn( - (async move { - let mut has_been_killed = false; - - //if we have been killed, we want to drain the queue of jobs - while let Some(sr) = if has_been_killed && same_worker_queue_size2.load(Ordering::SeqCst) == 0 { job_completed_rx.try_recv().ok() } else { job_completed_rx.recv().await }{ - match sr { - SendResult::JobCompleted(jc) => { - #[cfg(feature = "prometheus")] - if let Some(wj) = worker_job_completed_channel_queue2.as_ref() { - wj.dec(); - } - let rsmq2 = rsmq2.clone(); - - let is_init_script_and_failure = !jc.success && jc.job.tag.as_str() == INIT_SCRIPT_TAG; - let is_dependency_job = matches!( - jc.job.job_kind, - JobKind::Dependencies | JobKind::FlowDependencies - ); - handle_receive_completed_job( - jc, - &base_internal_url2, - &db2, - &worker_dir2, - &same_worker_tx2, - rsmq2, - &worker_name2, - worker_save_completed_job_duration2.clone(), - worker_flow_transition_duration2.clone(), - job_completed_sender.clone(), - ) - .await; - if is_init_script_and_failure { - tracing::error!("init script errored, exiting"); - killpill_tx2.send(()).unwrap_or_default(); - break; - } - if is_dependency_job && is_dedicated_worker { - tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted."); - sqlx::query!( - "UPDATE config SET config = config WHERE name = $1", - format!("worker__{}", *WORKER_GROUP) - ) - .execute(&db2) - .await - .expect("update config to trigger restart of all dedicated workers at that config"); - killpill_tx2.send(()).unwrap_or_default(); - } - } - SendResult::UpdateFlow { - flow, - w_id, - success, - result, - worker_dir, - stop_early_override, - token, - } => { - // let r; - tracing::info!(parent_flow = %flow, "updating flow status"); - if let Err(e) = update_flow_status_after_job_completion( - &db2, - &AuthedClient { - base_internal_url: base_internal_url2.to_string(), - workspace: w_id.clone(), - token: token.clone(), - force_client: None, - }, - flow, - &Uuid::nil(), - &w_id, - success, - Arc::new(result), - true, - same_worker_tx2.clone(), - &worker_dir, - stop_early_override, - rsmq2.clone(), - &worker_name2, - job_completed_sender.clone(), - ) - .await - { - tracing::error!("Error updating flow status after job completion for {flow} on {worker_name2}: {e:#}"); - } - } - SendResult::Kill => { - has_been_killed = true; - } - } - } - - job_completed_processor_is_done2.store(true, Ordering::SeqCst); - tracing::info!("finished processing all completed jobs"); - }) - .instrument(tracing::Span::current()), + let send_result = start_background_processor( + job_completed_rx, + job_completed_tx.0.clone(), + same_worker_queue_size.clone(), + job_completed_processor_is_done.clone(), + base_internal_url.to_string(), + db.clone(), + worker_dir.clone(), + same_worker_tx.clone(), + rsmq.clone(), + worker_name.clone(), + killpill_tx.clone(), + is_dedicated_worker, ); let mut last_executed_job: Option = None; - let mut last_checked_suspended = Instant::now(); #[cfg(feature = "benchmark")] let mut started = false; #[cfg(feature = "benchmark")] - let mut infos = BenchmarkInfo { iters: 0, timings: vec![] }; + let mut infos = BenchmarkInfo::new(); let vacuum_shift = rand::thread_rng().gen_range(0..VACUUM_PERIOD); @@ -1288,9 +1001,6 @@ pub async fn run_worker>, ) = (HashMap::new(), false, vec![]); - #[cfg(feature = "benchmark")] - tracing::info!("pre loop time {}s", start.elapsed().as_secs_f64()); - if i_worker == 1 { if let Err(e) = queue_init_bash_maybe(db, same_worker_tx.clone(), &worker_name, rsmq.clone()).await @@ -1321,14 +1031,13 @@ pub async fn run_worker = vec![false; 30]; + let mut last_suspend_first = Instant::now(); let mut killed_but_draining_same_worker_jobs = false; + loop { #[cfg(feature = "benchmark")] - let loop_start = Instant::now(); - - #[cfg(feature = "benchmark")] - let mut timing = vec![]; + let mut bench = BenchmarkIter::new(); #[cfg(feature = "prometheus")] if let Some(wk) = worker_busy.as_ref() { @@ -1336,6 +1045,8 @@ pub async fn run_worker NUM_SECS_READINGS { @@ -1360,16 +1072,24 @@ pub async fn run_worker 0 && infos.iters == benchmark_jobs as u64 { + tracing::info!("benchmark finished, exiting"); + job_completed_tx + .0 + .send(SendResult::Kill) + .await + .expect("send kill to job completed tx"); + break; + } else { + tracing::info!("benchmark not finished, still pulling jobs {}", infos.iters); + } + let next_job = { // println!("2: {:?}", instant.elapsed()); #[cfg(feature = "benchmark")] @@ -1432,7 +1165,11 @@ pub async fn run_worker 3 { - last_checked_suspended = Instant::now(); - true - } else { - false - }; + let likelihood_of_suspend = + (1.0 + last_30jobs_suspended.iter().filter(|&&x| x).count() as f64) / 31.0; + let suspend_first = suspend_first_success + || rand::random::() < likelihood_of_suspend + || last_suspend_first.elapsed().as_secs_f64() > 5.0; + + if suspend_first { + last_suspend_first = Instant::now(); + } + let job = pull(&db, rsmq.clone(), suspend_first).await; - add_time!(timing, loop_start, "post pull"); + + add_time!(bench, "job pulled from DB"); let duration_pull_s = pull_time.elapsed().as_secs_f64(); let err_pull = job.is_ok(); - let empty = job.as_ref().is_ok_and(|x| x.is_none()); - suspend_first_success = suspend_first && !empty; + // let empty = job.as_ref().is_ok_and(|x| x.is_none()); + if !agent_mode && duration_pull_s > 0.5 { + let empty = job.as_ref().is_ok_and(|x| x.0.is_none()); tracing::warn!("pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}"); #[cfg(feature = "prometheus")] if empty { @@ -1479,6 +1225,7 @@ pub async fn run_worker 0.1 { + let empty = job.as_ref().is_ok_and(|x| x.0.is_none()); tracing::warn!("pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}"); #[cfg(feature = "prometheus")] if empty { @@ -1490,9 +1237,17 @@ pub async fn run_worker 30 { + last_30jobs_suspended.remove(0); + } + } + suspend_first_success = suspend_first && suspend_success; + #[cfg(feature = "prometheus")] + if j.0.is_some() { if let Some(wp) = worker_pull_duration_counter.as_ref() { wp.inc_by(duration_pull_s); } @@ -1508,7 +1263,7 @@ pub async fn run_worker { - - last_executed_job = None; jobs_executed += 1; tracing::debug!("started handling of job {}", job.id); - if matches!(job.job_kind, JobKind::Script | JobKind::Preview) { - if !dedicated_workers.is_empty() { let key_o = if is_flow_worker { job.flow_step_id.as_ref().map(|x| x.to_string()) @@ -1539,43 +1291,23 @@ pub async fn run_worker { handle_job_error( - db, - &authed_client.get_authed().await, - arc_job.as_ref(), - 0, - None, - err, - false, - same_worker_tx.clone(), - &worker_dir, - rsmq.clone(), - &worker_name, - (&job_completed_tx.0).clone(), - ) - .await; - if is_init_script { - tracing::error!("init script job failed (in handler), exiting"); - update_worker_ping_for_failed_init_script(db, &worker_name, arc_job.id).await; + db, + &authed_client.get_authed().await, + arc_job.as_ref(), + 0, + None, + err, + false, + same_worker_tx.clone(), + &worker_dir, + rsmq.clone(), + &worker_name, + (&job_completed_tx.0).clone(), + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await; + if is_init_script { + tracing::error!("init script job failed (in handler), exiting"); + update_worker_ping_for_failed_init_script( + db, + &worker_name, + arc_job.id, + ) + .await; + break; + } + } + Ok(false) if is_init_script => { + tracing::error!("init script job failed, exiting"); + update_worker_ping_for_failed_init_script(db, &worker_name, arc_job.id) + .await; break; } - }, - Ok(false) if is_init_script => { - tracing::error!("init script job failed, exiting"); - update_worker_ping_for_failed_init_script(db, &worker_name, arc_job.id).await; - break; - + _ => {} } - _ => {} - } #[cfg(feature = "prometheus")] if let Some(duration) = _timer.map(|x| x.stop_and_record()) { @@ -1775,12 +1512,12 @@ pub async fn run_worker( @@ -1912,58 +1648,6 @@ async fn queue_init_bash_maybe<'c, R: rsmq_async::RsmqConnection + Send + 'c>( } } -// async fn process_result( -// client: AuthedClient, -// job: QueuedJob, -// result: error::Result, -// cached_res_path: Option, -// db: &DB, -// worker_dir: &str, -// job_dir: &str, -// metrics: Option, -// same_worker_tx: Sender, -// base_internal_url: &str, -// rsmq: Option, -// job_completed_tx: Sender, -// logs: String, -// ) -> error::Result<()> { - -// fn build_language_metrics( -// worker_execution_failed: &HashMap< -// Option, -// prometheus::core::GenericCounter, -// >, -// language: &Option, -// ) -> Option { -// let metrics = if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { -// Some(Metrics { -// worker_execution_failed: worker_execution_failed -// .get(language) -// .expect("no timer found") -// .clone(), -// }) -// } else { -// None -// }; -// metrics -// } - -// pub async fn create_barrier_for_all_workers(num_workers: u32, sync_barrier: Arc>>) { -// tracing::debug!("acquiring write lock"); -// let mut barrier = sync_barrier.write().await; -// *barrier = Some(tokio::sync::Barrier::new(num_workers as usize)); -// drop(barrier); -// tracing::debug!("dropped write lock"); -// if let Some(b) = sync_barrier.read().await.as_ref() { -// tracing::debug!("leader worker waiting for barrier"); -// b.wait().await; -// tracing::debug!("leader worker done waiting for barrier"); -// }; -// let mut barrier = sync_barrier.write().await; -// *barrier = None; -// tracing::debug!("leader worker done waiting for"); -// } - pub enum SendResult { JobCompleted(JobCompleted), UpdateFlow { @@ -1978,20 +1662,6 @@ pub enum SendResult { Kill, } -// db: &DB, -// client: &AuthedClient, -// flow: uuid::Uuid, -// job_id_for_status: &Uuid, -// w_id: &str, -// success: bool, -// result: &'a RawValue, -// unrecoverable: bool, -// same_worker_tx: Sender, -// worker_dir: &str, -// stop_early_override: Option, -// rsmq: Option, -// worker_name: &str, - #[derive(Debug, Clone)] pub struct JobCompleted { pub job: Arc, @@ -2012,6 +1682,7 @@ async fn do_nativets( mem_peak: &mut i32, canceled_by: &mut Option, worker_name: &str, + occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result<(Box, String)> { let args = build_args_map(job, client, db).await?.map(Json); let job_args = if args.is_some() { @@ -2033,6 +1704,7 @@ async fn do_nativets( worker_name, &job.workspace_id, true, + occupancy_metrics, ) .await?; Ok((result.0, result.1)) @@ -2057,9 +1729,8 @@ async fn handle_queued_job( base_internal_url: &str, rsmq: Option, job_completed_tx: JobCompletedSender, - worker_code_execution_metric: &mut f32, - _worker_flow_initial_transition_duration: Option, - _worker_code_execution_duration: Option, + occupancy_metrics: &mut OccupancyMetrics, + #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { if job.canceled { return Err(Error::JsonErr(canceled_job_to_result(&job))); @@ -2156,7 +1827,11 @@ async fn handle_queued_job( ) .fetch_one(db) .await - .map_err(|e| Error::InternalErr(format!("Fetching script path from queue for caching purposes: {e:#}")))? + .map_err(|e| { + Error::InternalErr(format!( + "Fetching script path from queue for caching purposes: {e:#}" + )) + })? .ok_or_else(|| Error::InternalErr(format!("Expected script_path")))?; let step = match step.unwrap() { Step::Step(i) => i.to_string(), @@ -2273,6 +1948,7 @@ async fn handle_queued_job( base_internal_url, &client.get_token().await, rsmq.clone(), + occupancy_metrics, ) .await } @@ -2288,6 +1964,7 @@ async fn handle_queued_job( base_internal_url, &client.get_token().await, rsmq.clone(), + occupancy_metrics, ) .await } @@ -2302,6 +1979,7 @@ async fn handle_queued_job( base_internal_url, &client.get_token().await, rsmq.clone(), + occupancy_metrics, ) .await .map(|()| serde_json::from_str("{}").unwrap()), @@ -2313,8 +1991,6 @@ async fn handle_queued_job( .map(|x| x.to_owned()) .unwrap_or_else(|| serde_json::from_str("{}").unwrap())), _ => { - #[cfg(feature = "prometheus")] - let timer = _worker_code_execution_duration.map(|x| x.start_timer()); let metric_timer = Instant::now(); let r = handle_code_execution_job( job.as_ref(), @@ -2328,11 +2004,11 @@ async fn handle_queued_job( worker_name, &mut column_order, &mut new_args, + occupancy_metrics, ) .await; - *worker_code_execution_metric += metric_timer.elapsed().as_secs_f32(); - #[cfg(feature = "prometheus")] - timer.map(|x| x.stop_and_record()); + occupancy_metrics.total_duration_of_running_jobs += + metric_timer.elapsed().as_secs_f32(); r } }; @@ -2366,7 +2042,6 @@ async fn handle_queued_job( } } - pub fn build_envs( envs: Option>, ) -> windmill_common::error::Result> { @@ -2496,6 +2171,7 @@ async fn handle_code_execution_job( worker_name: &str, column_order: &mut Option>, new_args: &mut Option>>, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { let ContentReqLangEnvs { content: inner_content, @@ -2551,6 +2227,7 @@ async fn handle_code_execution_job( canceled_by, worker_name, column_order, + occupancy_metrics, ) .await; } else if language == Some(ScriptLang::Mysql) { @@ -2563,6 +2240,7 @@ async fn handle_code_execution_job( canceled_by, worker_name, column_order, + occupancy_metrics, ) .await; } else if language == Some(ScriptLang::Bigquery) { @@ -2584,6 +2262,7 @@ async fn handle_code_execution_job( canceled_by, worker_name, column_order, + occupancy_metrics, ) .await; } @@ -2606,6 +2285,7 @@ async fn handle_code_execution_job( canceled_by, worker_name, column_order, + occupancy_metrics, ) .await; } @@ -2627,6 +2307,7 @@ async fn handle_code_execution_job( mem_peak, canceled_by, worker_name, + occupancy_metrics, ) .await; } @@ -2639,6 +2320,7 @@ async fn handle_code_execution_job( mem_peak, canceled_by, worker_name, + occupancy_metrics, ) .await; } else if language == Some(ScriptLang::Nativets) { @@ -2668,6 +2350,7 @@ async fn handle_code_execution_job( mem_peak, canceled_by, worker_name, + occupancy_metrics, ) .await?; append_logs(&job.id, &job.workspace_id, ts_logs, db).await; @@ -2688,7 +2371,11 @@ async fn handle_code_execution_job( ); let shared_mount = if job.same_worker && job.language != Some(ScriptLang::Deno) { - let folder = if job.language == Some(ScriptLang::Go) { "/go" } else { "" }; + let folder = if job.language == Some(ScriptLang::Go) { + "/go" + } else { + "" + }; format!( r#" mount {{ @@ -2729,6 +2416,7 @@ mount {{ base_internal_url, envs, new_args, + occupancy_metrics, ) .await } @@ -2746,6 +2434,7 @@ mount {{ worker_name, envs, new_args, + occupancy_metrics, ) .await } @@ -2765,6 +2454,7 @@ mount {{ envs, &shared_mount, new_args, + occupancy_metrics, ) .await } @@ -2782,6 +2472,7 @@ mount {{ base_internal_url, worker_name, envs, + occupancy_metrics, ) .await } @@ -2798,6 +2489,7 @@ mount {{ base_internal_url, worker_name, envs, + occupancy_metrics, ) .await } @@ -2814,6 +2506,7 @@ mount {{ base_internal_url, worker_name, envs, + occupancy_metrics, ) .await } @@ -2831,6 +2524,7 @@ mount {{ worker_name, envs, &shared_mount, + occupancy_metrics, ) .await } @@ -2848,11 +2542,11 @@ mount {{ base_internal_url, worker_name, envs, + occupancy_metrics, ) .await } Some(ScriptLang::Ansible) => { - handle_ansible_job( requirements_o, job_dir, @@ -2867,7 +2561,9 @@ mount {{ &shared_mount, base_internal_url, envs, - ).await + occupancy_metrics, + ) + .await } _ => panic!("unreachable, language is not supported: {language:#?}"), }; diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 8c1161cc3e..6474af6d46 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -29,7 +29,10 @@ use sqlx::FromRow; use tokio::sync::mpsc::Sender; use tracing::instrument; use uuid::Uuid; +use windmill_common::add_time; use windmill_common::auth::JobPerms; +#[cfg(feature = "benchmark")] +use windmill_common::bench::BenchmarkIter; use windmill_common::db::Authed; use windmill_common::flow_status::{ ApprovalConditions, FlowStatusModuleWParent, Iterator, JobResult, @@ -50,8 +53,8 @@ use windmill_common::{ }; use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ - add_completed_job, add_completed_job_error, append_logs, get_queued_job, - handle_maybe_scheduled_job, CanceledBy, PushArgs, PushIsolationLevel, WrappedError, + add_completed_job, add_completed_job_error, append_logs, handle_maybe_scheduled_job, + CanceledBy, PushArgs, PushIsolationLevel, WrappedError, }; type DB = sqlx::Pool; @@ -76,6 +79,7 @@ pub async fn update_flow_status_after_job_completion< rsmq: Option, worker_name: &str, job_completed_tx: Sender, + #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> error::Result<()> { // this is manual tailrecursion because async_recursion blows up the stack // todo!(); @@ -97,6 +101,8 @@ pub async fn update_flow_status_after_job_completion< rsmq.clone(), worker_name, job_completed_tx.clone(), + #[cfg(feature = "benchmark")] + bench, ) .await?; while let Some(nrec) = rec { @@ -117,6 +123,8 @@ pub async fn update_flow_status_after_job_completion< rsmq.clone(), worker_name, job_completed_tx.clone(), + #[cfg(feature = "benchmark")] + bench, ) .await { @@ -141,6 +149,8 @@ pub async fn update_flow_status_after_job_completion< rsmq.clone(), worker_name, job_completed_tx.clone(), + #[cfg(feature = "benchmark")] + bench, ) .await? } @@ -158,10 +168,7 @@ pub struct RecUpdateFlowStatusAfterJobCompletion { } #[derive(FromRow)] -pub struct SkipIfStopped { - pub skip_if_stopped: Option, - pub stop_early_expr: Option, - pub continue_on_error: Option, +pub struct RowArgs { pub args: Option>>>, } @@ -173,6 +180,7 @@ struct RecoveryObject { #[derive(sqlx::FromRow, Deserialize)] pub struct RowFlowStatus { pub flow_status: sqlx::types::Json>, + pub current_module: Option>>, } // #[instrument(level = "trace", skip_all)] pub async fn update_flow_status_after_job_completion_internal< @@ -193,7 +201,9 @@ pub async fn update_flow_status_after_job_completion_internal< rsmq: Option, worker_name: &str, job_completed_tx: Sender, + #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> error::Result> { + add_time!(bench, "update flow status internal START"); let ( should_continue_flow, flow_job, @@ -206,7 +216,7 @@ pub async fn update_flow_status_after_job_completion_internal< // tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}"); let old_status_json = sqlx::query_as::<_, RowFlowStatus>( - "SELECT flow_status FROM queue WHERE id = $1 AND workspace_id = $2", + "SELECT flow_status, raw_flow->'modules'->(flow_status->'step')::int as current_module FROM queue WHERE id = $1 AND workspace_id = $2", ) .bind(flow) .bind(w_id) @@ -225,6 +235,16 @@ pub async fn update_flow_status_after_job_completion_internal< ))) })?; + let current_module = if let Some(x) = old_status_json.current_module { + Some(serde_json::from_str::(x.0.get()).or_else(|e| { + Err(Error::InternalErr(format!( + "requiring current module to be parsable as FlowModule: {e:?}" + ))) + })?) + } else { + None + }; + let module_step = Step::from_i32_and_len(old_status.step, old_status.modules.len()); let module_status = match module_step { @@ -247,9 +267,17 @@ pub async fn update_flow_status_after_job_completion_internal< module_status, FlowStatusModule::InProgress { iterator: Some(_), .. } ) { - let (loop_failures, parallelism) = - compute_skip_loop_failures_and_parallelism(flow, old_status.step, db).await?; - (true, loop_failures.unwrap_or(false), parallelism) + let value = current_module + .as_ref() + .and_then(|x| x.get_value_with_skip_failures().ok()); + ( + true, + value + .as_ref() + .and_then(|x| x.skip_failures) + .unwrap_or(false), + value.as_ref().and_then(|x| x.parallelism), + ) } else { (false, false, None) }; @@ -267,37 +295,43 @@ pub async fn update_flow_status_after_job_completion_internal< stop_early_override { //do not stop early if module is a flow step - let flow_job = get_queued_job(&flow, w_id, db) - .await? - .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?; - let module = get_module(&flow_job, &module_step); + let step = match module_step { + Step::PreprocessorStep => None, + Step::FailureStep => None, + Step::Step(i) => Some(i), + }; - if module.is_some_and(|x| x.is_flow()) { + let is_flow = if let Some(step) = step { + sqlx::query_scalar!( + "SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2", + step as i32, + &flow + ) + .fetch_one(db) + .await + .map_err(|e| { + Error::InternalErr(format!("error during retrieval of step's type: {e:#}")) + })? + .unwrap_or(false) + } else { + false + }; + + if is_flow { (false, false, false) } else { (true, se, false) } } else if is_failure_step || matches!(module_step, Step::PreprocessorStep) { (false, false, false) - } else { - let r = sqlx::query_as::<_, SkipIfStopped>( - "SELECT - raw_flow->'modules'->$1::int->'stop_after_if'->>'expr' as stop_early_expr, - (raw_flow->'modules'->$1::int->'stop_after_if'->>'skip_if_stopped')::bool as skip_if_stopped, - (raw_flow->'modules'->$1::int->'continue_on_error')::bool as continue_on_error, - args - FROM queue - WHERE id = $2" - ) - .bind(old_status.step) - .bind(flow) - .fetch_one(db) - .await - .map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e:#}")))?; - + } else if let Some(current_module) = current_module.as_ref() { let stop_early = success && !is_branch_all - && if let Some(expr) = r.stop_early_expr.clone() { + && if let Some(ref expr) = current_module + .stop_after_if + .as_ref() + .map(|x| x.expr.clone()) + { let all_iters = match &module_status { FlowStatusModule::InProgress { flow_jobs: Some(flow_jobs), .. } if expr.contains("all_iters") => @@ -308,14 +342,22 @@ pub async fn update_flow_status_after_job_completion_internal< } _ => None, }; + let args = sqlx::query_as::<_, RowArgs>( + "SELECT + args + FROM queue + WHERE id = $2", + ) + .bind(old_status.step) + .bind(flow) + .fetch_one(db) + .await + .map_err(|e| { + Error::InternalErr(format!("retrieval of args from state: {e:#}")) + })?; compute_bool_from_expr( - expr, - Marc::new( - r.args - .map(|x| x.0) - .unwrap_or_else(|| serde_json::from_str("{}").unwrap()) - .to_owned(), - ), + expr.to_string(), + Marc::new(args.args.unwrap_or_default().0), result.clone(), all_iters, None, @@ -329,9 +371,15 @@ pub async fn update_flow_status_after_job_completion_internal< }; ( stop_early, - r.skip_if_stopped.unwrap_or(false), - r.continue_on_error.unwrap_or(false), + current_module + .stop_after_if + .as_ref() + .map(|x| x.skip_if_stopped) + .unwrap_or(false), + current_module.continue_on_error.unwrap_or(false), ) + } else { + (false, false, false) }; let skip_branch_failure = match module_status { @@ -340,26 +388,23 @@ pub async fn update_flow_status_after_job_completion_internal< parallel, .. } => compute_skip_branchall_failure( - flow, job_id_for_status, - old_status.step, *branch, *parallel, db, + current_module.as_ref(), ) .await? .unwrap_or(false), _ => false, }; - let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into(); - if matches!(module_step, Step::PreprocessorStep) { sqlx::query!( "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2", job_id_for_status, flow - ).execute(&mut tx).await.map_err(|e| { + ).execute(db).await.map_err(|e| { Error::InternalErr(format!("error while updating args in preprocessing step: {e:#}")) })?; @@ -367,7 +412,7 @@ pub async fn update_flow_status_after_job_completion_internal< r#"UPDATE completed_job SET args = '{"reason":"PREPROCESSOR_ARGS_ARE_DISCARDED"}'::jsonb WHERE id = $1"#, job_id_for_status ) - .execute(&mut tx) + .execute(db) .await .map_err(|e| { Error::InternalErr(format!( @@ -376,6 +421,10 @@ pub async fn update_flow_status_after_job_completion_internal< })?; } + let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into(); + + add_time!(bench, "process module status START"); + let (inc_step_counter, new_status) = match module_status { FlowStatusModule::InProgress { iterator, @@ -387,25 +436,38 @@ pub async fn update_flow_status_after_job_completion_internal< } if *parallel => { let (nindex, len) = match (iterator, branchall) { (Some(Iterator { itered, .. }), _) => { - set_success_in_flow_job_success( - flow_jobs_success, - jobs, - job_id_for_status, - &old_status, - flow, - success, - &mut tx, - ) - .await?; + let position = if flow_jobs_success.is_some() { + find_flow_job_index(jobs, job_id_for_status) + } else { + None + }; - let nindex = sqlx::query_scalar!( + let nindex = if let Some(position) = position { + sqlx::query_scalar!( "UPDATE queue - SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb) + SET flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), + ARRAY['modules', $1::TEXT, 'iterator', 'index'], + ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb + ), + last_ping = NULL WHERE id = $2 RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", old_status.step, - flow - ) + flow, + position as i32, + json!(success) + )} else { + sqlx::query_scalar!( + "UPDATE queue + SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb), + last_ping = NULL + WHERE id = $2 + RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + old_status.step, + flow + ) + } .fetch_one(&mut tx) .await.map_err(|e| { Error::InternalErr(format!( @@ -421,24 +483,35 @@ pub async fn update_flow_status_after_job_completion_internal< (nindex, itered.len() as i32) } (_, Some(BranchAllStatus { len, .. })) => { - set_success_in_flow_job_success( - flow_jobs_success, - jobs, - job_id_for_status, - &old_status, - flow, - success, - &mut tx, - ) - .await?; - let nindex = sqlx::query_scalar!( + let position = if flow_jobs_success.is_some() { + find_flow_job_index(jobs, job_id_for_status) + } else { + None + }; + + let nindex = if let Some(position) = position { + sqlx::query_scalar!( + "UPDATE queue + SET flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), + ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb), + last_ping = NULL + WHERE id = $2 + RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + old_status.step, + flow, + position as i32, + json!(success) + ) + } else { sqlx::query_scalar!( "UPDATE queue - SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb) + SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb), + last_ping = NULL WHERE id = $2 RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", old_status.step, flow - ) + )} .fetch_one(&mut tx) .await .map_err(|e| { @@ -517,9 +590,9 @@ pub async fn update_flow_status_after_job_completion_internal< tracing::info!( "parallel iteration {job_id_for_status} of flow {flow} has finished", ); - (true, Some(new_status)) } else { + add_time!(bench, "handle parallel flow start"); tx.commit().await?; if parallelism.is_some() { @@ -537,18 +610,6 @@ pub async fn update_flow_status_after_job_completion_internal< })?; } - sqlx::query!( - "UPDATE queue - SET last_ping = null - WHERE id = $1", - flow - ) - .execute(db) - .await - .map_err(|e| { - Error::InternalErr(format!("error while setting last ping to null: {e:#}")) - })?; - let r = sqlx::query_scalar!( "DELETE FROM parallel_monitor_lock WHERE parent_flow_id = $1 and job_id = $2 RETURNING last_ping", flow, @@ -562,7 +623,7 @@ pub async fn update_flow_status_after_job_completion_internal< r.unwrap() ); } - + add_time!(bench, "non final parallel flow finished"); return Ok(None); } } @@ -652,25 +713,12 @@ pub async fn update_flow_status_after_job_completion_internal< ) } else { let inc = if continue_on_error { - let retry = sqlx::query_scalar!( - "SELECT raw_flow->'modules'->$2::int->'retry' FROM queue WHERE id = $1", - flow, - old_status.step - ) - .fetch_optional(&mut tx) - .await - .map_err(|e| { - Error::InternalErr(format!( - "error while getting retry from step: {e:#}" - )) - })? - .flatten(); - - let retry = retry - .map(|x| serde_json::from_value::(x).ok()) - .flatten() + let retry = current_module + .as_ref() + .and_then(|x| x.retry.clone()) .unwrap_or_default(); - tracing::info!("update flow status on rety: {retry:#?} "); + + tracing::info!("update flow status on rety: {retry:#?} "); next_retry(&retry, &old_status.retry).is_none() } else { false @@ -802,30 +850,27 @@ pub async fn update_flow_status_after_job_completion_internal< match &new_status { Some(FlowStatusModule::Success { .. }) if is_loop || is_branch_all => { - let r_after_all_iters = sqlx::query_as::<_, SkipIfStopped>( - "SELECT - raw_flow->'modules'->$1::int->'stop_after_all_iters_if'->>'expr' as stop_early_expr, - (raw_flow->'modules'->$1::int->'stop_after_all_iters_if'->>'skip_if_stopped')::bool as skip_if_stopped, - NULL as continue_on_error, + if let Some(ref expr) = current_module + .as_ref() + .and_then(|m| m.stop_after_all_iters_if.as_ref().map(|x| x.expr.clone())) + { + let args = sqlx::query_as::<_, RowArgs>( + "SELECT args FROM queue - WHERE id = $2" + WHERE id = $2", ) .bind(old_status.step) .bind(flow) .fetch_one(db) .await - .map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e:#}")))?; - if let Some(expr) = r_after_all_iters.stop_early_expr { + .map_err(|e| { + Error::InternalErr(format!("retrieval of args from state: {e:#}")) + })?; + let should_stop = compute_bool_from_expr( - expr, - Marc::new( - r_after_all_iters - .args - .map(|x| x.0) - .unwrap_or_else(|| serde_json::from_str("{}").unwrap()) - .to_owned(), - ), + expr.to_string(), + Marc::new(args.args.unwrap_or_default().0), nresult.clone(), None, None, @@ -837,7 +882,14 @@ pub async fn update_flow_status_after_job_completion_internal< if should_stop { stop_early = should_stop; - skip_if_stop_early = r_after_all_iters.skip_if_stopped.unwrap_or(false); + skip_if_stop_early = current_module + .as_ref() + .and_then(|m| { + m.stop_after_all_iters_if + .as_ref() + .map(|x| x.skip_if_stopped) + }) + .unwrap_or(false); } } } @@ -862,6 +914,7 @@ pub async fn update_flow_status_after_job_completion_internal< let flow_job = get_queued_job_tx(flow, w_id, tx.transaction_mut()) .await? .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?; + tx.commit().await?; let job_root = flow_job .root_job @@ -894,14 +947,13 @@ pub async fn update_flow_status_after_job_completion_internal< false if !is_failure_step && !skip_error_handler - && has_failure_module(flow, tx.transaction_mut()).await? => + && has_failure_module(flow, db).await? => { true } false => false, }; - tx.commit().await?; tracing::debug!(id = %flow_job.id, root_id = %job_root, "flow status updated"); ( @@ -964,6 +1016,8 @@ pub async fn update_flow_status_after_job_completion_internal< rsmq.clone(), worker_name, true, + #[cfg(feature = "benchmark")] + bench, ) .await?; } else { @@ -994,6 +1048,8 @@ pub async fn update_flow_status_after_job_completion_internal< let success = success && (!is_failure_step || result_has_recover_true(nresult.clone())) && !skip_error_handler; + + add_time!(bench, "flow status update 1"); if success { add_completed_job( db, @@ -1005,6 +1061,8 @@ pub async fn update_flow_status_after_job_completion_internal< None, rsmq.clone(), true, + #[cfg(feature = "benchmark")] + bench, ) .await?; } else { @@ -1022,6 +1080,8 @@ pub async fn update_flow_status_after_job_completion_internal< None, rsmq.clone(), true, + #[cfg(feature = "benchmark")] + bench, ) .await?; } @@ -1059,6 +1119,8 @@ pub async fn update_flow_status_after_job_completion_internal< rsmq.clone(), worker_name, true, + #[cfg(feature = "benchmark")] + bench, ) .await; true @@ -1096,6 +1158,10 @@ pub async fn update_flow_status_after_job_completion_internal< } } +fn find_flow_job_index(flow_jobs: &Vec, job_id_for_status: &Uuid) -> Option { + flow_jobs.iter().position(|x| x == job_id_for_status) +} + async fn set_success_in_flow_job_success<'c, R: rsmq_async::RsmqConnection + Send>( flow_jobs_success: &Option>>, flow_jobs: &Vec, @@ -1105,10 +1171,8 @@ async fn set_success_in_flow_job_success<'c, R: rsmq_async::RsmqConnection + Sen success: bool, tx: &mut QueueTransaction<'c, R>, ) -> error::Result<()> { - let flow_jobs_success = flow_jobs_success.clone(); - if flow_jobs_success.is_some() { - let position = flow_jobs.iter().position(|x| x == job_id_for_status); + let position = find_flow_job_index(flow_jobs, job_id_for_status); if let Some(position) = position { sqlx::query!( "UPDATE queue SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4) WHERE id = $2", @@ -1172,31 +1236,12 @@ fn get_module(flow_job: &QueuedJob, module_step: &Step) -> Option { } } -async fn compute_skip_loop_failures_and_parallelism( - flow: Uuid, - step: i32, - db: &DB, -) -> Result<(Option, Option), Error> { - sqlx::query_as( - "SELECT (raw_flow->'modules'->$1->'value'->>'skip_failures')::bool, (raw_flow->'modules'->$1->'value'->>'parallelism')::int - FROM queue - WHERE id = $2", - ) - .bind(step) - .bind(flow) - .fetch_one(db) - .await - .map(|(v, n)| (v,n)) - .map_err(|e| Error::InternalErr(format!("error during retrieval of skip_loop_failures: {e:#}"))) -} - async fn compute_skip_branchall_failure<'c>( - flow: Uuid, job: &Uuid, - step: i32, branch: usize, parallel: bool, db: &DB, + flow_module: Option<&FlowModule>, ) -> Result, Error> { let branch = if parallel { sqlx::query_scalar!("SELECT script_path FROM completed_job WHERE id = $1", job) @@ -1220,35 +1265,23 @@ async fn compute_skip_branchall_failure<'c>( } else { branch as i32 }; - sqlx::query_as( - "SELECT (raw_flow->'modules'->$1->'value'->'branches'->$2->>'skip_failure')::bool - FROM queue - WHERE id = $3", - ) - .bind(step) - .bind(branch) - .bind(flow) - .fetch_one(db) - .await - .map(|(v,)| v) - .map_err(|e| { - Error::InternalErr(format!( - "error during retrieval of skip_loop_failures: {e:#}" - )) - }) + Ok(flow_module + .and_then(|x| x.get_branches_skip_failures().ok()) + .and_then(|x| { + x.branches + .get(branch as usize) + .map(|x| x.skip_failure.unwrap_or(false)) + })) } -async fn has_failure_module<'c>( - flow: Uuid, - tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, -) -> Result { +async fn has_failure_module<'c>(flow: Uuid, db: &DB) -> Result { sqlx::query_scalar::<_, Option>( "SELECT raw_flow->'failure_module' != 'null'::jsonb FROM queue WHERE id = $1", ) .bind(flow) - .fetch_one(&mut **tx) + .fetch_one(db) .await .map_err(|e| { Error::InternalErr(format!( diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 6381dac896..6f25ec606b 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -25,6 +25,7 @@ use windmill_parser_py_imports::parse_relative_imports; use windmill_parser_ts::parse_expr_for_imports; use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel}; +use crate::common::OccupancyMetrics; use crate::python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}; use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; use crate::{ @@ -212,6 +213,7 @@ pub async fn handle_dependency_job, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { let raw_code = match job.raw_code { Some(ref code) => code.to_owned(), @@ -273,6 +275,7 @@ pub async fn handle_dependency_job, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { let job_path = job.script_path.clone().ok_or_else(|| { error::Error::InternalErr( @@ -605,6 +609,7 @@ pub async fn handle_flow_dependency_job( base_internal_url: &str, token: &str, locks_to_reload: &Option>, + occupancy_metrics: &mut OccupancyMetrics, // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) ) ) -> Result<( Vec, @@ -754,6 +760,7 @@ async fn lock_modules<'c>( base_internal_url, token, locks_to_reload, + occupancy_metrics, )) .await?; e.value = FlowModuleValue::ForloopFlow { @@ -785,6 +792,7 @@ async fn lock_modules<'c>( base_internal_url, token, locks_to_reload, + occupancy_metrics, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -809,6 +817,7 @@ async fn lock_modules<'c>( base_internal_url, token, locks_to_reload, + occupancy_metrics, )) .await?; e.value = @@ -835,6 +844,7 @@ async fn lock_modules<'c>( base_internal_url, token, locks_to_reload, + occupancy_metrics, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -856,6 +866,7 @@ async fn lock_modules<'c>( base_internal_url, token, locks_to_reload, + occupancy_metrics, )) .await?; e.value = FlowModuleValue::BranchOne { branches: nbranches, default: ndefault } @@ -904,6 +915,7 @@ async fn lock_modules<'c>( ), false, None, + occupancy_metrics, ) .await; // @@ -1014,6 +1026,7 @@ async fn lock_modules_app( job_path: &str, base_internal_url: &str, token: &str, + occupancy_metrics: &mut OccupancyMetrics, ) -> Result { match value { Value::Object(mut m) => { @@ -1058,6 +1071,7 @@ async fn lock_modules_app( &format!("{}/app", job.script_path()), false, None, + occupancy_metrics, ) .await; match new_lock { @@ -1113,6 +1127,7 @@ async fn lock_modules_app( job_path, base_internal_url, token, + occupancy_metrics, ) .await?, ); @@ -1135,6 +1150,7 @@ async fn lock_modules_app( job_path, base_internal_url, token, + occupancy_metrics, ) .await?, ); @@ -1156,6 +1172,7 @@ pub async fn handle_app_dependency_job, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result<()> { let job_path = job.script_path.clone().ok_or_else(|| { error::Error::InternalErr( @@ -1185,6 +1202,7 @@ pub async fn handle_app_dependency_job, ) -> std::result::Result { create_dependencies_dir(job_dir).await; let req: std::result::Result = pip_compile( @@ -1270,6 +1289,7 @@ async fn python_dep( db, worker_name, w_id, + occupancy_metrics, ) .await; // install the dependencies to pre-fill the cache @@ -1284,6 +1304,7 @@ async fn python_dep( worker_name, job_dir, worker_dir, + occupancy_metrics, ) .await; @@ -1313,6 +1334,7 @@ async fn capture_dependency_job( script_path: &str, raw_deps: bool, npm_mode: Option, + occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result { match job_language { ScriptLang::Python3 => { @@ -1342,6 +1364,7 @@ async fn capture_dependency_job( worker_name, w_id, worker_dir, + &mut Some(occupancy_metrics), ) .await } @@ -1364,6 +1387,7 @@ async fn capture_dependency_job( worker_name, w_id, worker_dir, + &mut Some(occupancy_metrics), ) .await } @@ -1385,6 +1409,7 @@ async fn capture_dependency_job( false, worker_name, w_id, + occupancy_metrics, ) .await } @@ -1404,6 +1429,7 @@ async fn capture_dependency_job( w_id, worker_name, base_internal_url, + &mut Some(occupancy_metrics), ) .await } @@ -1431,6 +1457,7 @@ async fn capture_dependency_job( None }, npm_mode, + &mut Some(occupancy_metrics), ) .await?; if req.is_some() && !raw_deps { @@ -1445,6 +1472,7 @@ async fn capture_dependency_job( base_internal_url, worker_name, &token, + occupancy_metrics, ) .await?; } @@ -1475,6 +1503,7 @@ async fn capture_dependency_job( worker_name, reqs, None, + occupancy_metrics, ) .await } @@ -1494,6 +1523,7 @@ async fn capture_dependency_job( db, worker_name, w_id, + occupancy_metrics, ) .await?; @@ -1507,6 +1537,7 @@ async fn capture_dependency_job( w_id, base_internal_url, &compute_rust_hash(&job_raw_code, Some(&lockfile)), + occupancy_metrics, ) .await?; Ok(lockfile) diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 22b8986a2a..2d88e6abed 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.401.0"; +export const VERSION = "v1.402.3"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/gen/core/OpenAPI.ts b/cli/gen/core/OpenAPI.ts index 88e02785ee..6af5246be2 100644 --- a/cli/gen/core/OpenAPI.ts +++ b/cli/gen/core/OpenAPI.ts @@ -54,7 +54,7 @@ export const OpenAPI: OpenAPIConfig = { PASSWORD: undefined, TOKEN: getEnv("WM_TOKEN"), USERNAME: undefined, - VERSION: '1.398.1', + VERSION: '1.401.0', WITH_CREDENTIALS: true, interceptors: { request: new Interceptors(), diff --git a/cli/gen/services.gen.ts b/cli/gen/services.gen.ts index 9894d6de36..ef6dafe997 100644 --- a/cli/gen/services.gen.ts +++ b/cli/gen/services.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise.ts'; import { OpenAPI } from './core/OpenAPI.ts'; import { request as __request } from './core/request.ts'; -import type { BackendVersionResponse, BackendUptodateResponse, GetLicenseIdResponse, GetOpenApiYamlResponse, GetAuditLogData, GetAuditLogResponse, ListAuditLogsData, ListAuditLogsResponse, LoginData, LoginResponse, LogoutResponse, GetUserData, GetUserResponse, UpdateUserData, UpdateUserResponse, IsOwnerOfPathData, IsOwnerOfPathResponse, SetPasswordData, SetPasswordResponse, CreateUserGloballyData, CreateUserGloballyResponse, GlobalUserUpdateData, GlobalUserUpdateResponse, GlobalUsernameInfoData, GlobalUsernameInfoResponse, GlobalUserRenameData, GlobalUserRenameResponse, GlobalUserDeleteData, GlobalUserDeleteResponse, GlobalUsersOverwriteData, GlobalUsersOverwriteResponse, GlobalUsersExportResponse, DeleteUserData, DeleteUserResponse, ListWorkspacesResponse, IsDomainAllowedResponse, ListUserWorkspacesResponse, ListWorkspacesAsSuperAdminData, ListWorkspacesAsSuperAdminResponse, CreateWorkspaceData, CreateWorkspaceResponse, ExistsWorkspaceData, ExistsWorkspaceResponse, ExistsUsernameData, ExistsUsernameResponse, GetGlobalData, GetGlobalResponse, SetGlobalData, SetGlobalResponse, GetLocalResponse, TestSmtpData, TestSmtpResponse, TestCriticalChannelsData, TestCriticalChannelsResponse, TestLicenseKeyData, TestLicenseKeyResponse, TestObjectStorageConfigData, TestObjectStorageConfigResponse, SendStatsResponse, GetLatestKeyRenewalAttemptResponse, RenewLicenseKeyData, RenewLicenseKeyResponse, CreateCustomerPortalSessionData, CreateCustomerPortalSessionResponse, TestMetadataData, TestMetadataResponse, ListGlobalSettingsResponse, GetCurrentEmailResponse, RefreshUserTokenResponse, GetTutorialProgressResponse, UpdateTutorialProgressData, UpdateTutorialProgressResponse, LeaveInstanceResponse, GetUsageResponse, GetRunnableResponse, GlobalWhoamiResponse, ListWorkspaceInvitesResponse, WhoamiData, WhoamiResponse, AcceptInviteData, AcceptInviteResponse, DeclineInviteData, DeclineInviteResponse, InviteUserData, InviteUserResponse, AddUserData, AddUserResponse, DeleteInviteData, DeleteInviteResponse, ArchiveWorkspaceData, ArchiveWorkspaceResponse, UnarchiveWorkspaceData, UnarchiveWorkspaceResponse, DeleteWorkspaceData, DeleteWorkspaceResponse, LeaveWorkspaceData, LeaveWorkspaceResponse, GetWorkspaceNameData, GetWorkspaceNameResponse, ChangeWorkspaceNameData, ChangeWorkspaceNameResponse, ChangeWorkspaceIdData, ChangeWorkspaceIdResponse, WhoisData, WhoisResponse, ExistsEmailData, ExistsEmailResponse, ListUsersAsSuperAdminData, ListUsersAsSuperAdminResponse, ListPendingInvitesData, ListPendingInvitesResponse, GetSettingsData, GetSettingsResponse, GetDeployToData, GetDeployToResponse, GetIsPremiumData, GetIsPremiumResponse, GetPremiumInfoData, GetPremiumInfoResponse, SetAutomaticBillingData, SetAutomaticBillingResponse, EditSlackCommandData, EditSlackCommandResponse, RunSlackMessageTestJobData, RunSlackMessageTestJobResponse, EditDeployToData, EditDeployToResponse, EditAutoInviteData, EditAutoInviteResponse, EditWebhookData, EditWebhookResponse, EditCopilotConfigData, EditCopilotConfigResponse, GetCopilotInfoData, GetCopilotInfoResponse, EditErrorHandlerData, EditErrorHandlerResponse, EditLargeFileStorageConfigData, EditLargeFileStorageConfigResponse, EditWorkspaceGitSyncConfigData, EditWorkspaceGitSyncConfigResponse, EditWorkspaceDeployUiSettingsData, EditWorkspaceDeployUiSettingsResponse, EditWorkspaceDefaultAppData, EditWorkspaceDefaultAppResponse, EditDefaultScriptsData, EditDefaultScriptsResponse, GetDefaultScriptsData, GetDefaultScriptsResponse, SetEnvironmentVariableData, SetEnvironmentVariableResponse, GetWorkspaceEncryptionKeyData, GetWorkspaceEncryptionKeyResponse, SetWorkspaceEncryptionKeyData, SetWorkspaceEncryptionKeyResponse, GetWorkspaceDefaultAppData, GetWorkspaceDefaultAppResponse, GetLargeFileStorageConfigData, GetLargeFileStorageConfigResponse, GetWorkspaceUsageData, GetWorkspaceUsageResponse, ListUsersData, ListUsersResponse, ListUsersUsageData, ListUsersUsageResponse, ListUsernamesData, ListUsernamesResponse, UsernameToEmailData, UsernameToEmailResponse, CreateTokenData, CreateTokenResponse, CreateTokenImpersonateData, CreateTokenImpersonateResponse, DeleteTokenData, DeleteTokenResponse, ListTokensData, ListTokensResponse, GetOidcTokenData, GetOidcTokenResponse, CreateVariableData, CreateVariableResponse, EncryptValueData, EncryptValueResponse, DeleteVariableData, DeleteVariableResponse, UpdateVariableData, UpdateVariableResponse, GetVariableData, GetVariableResponse, GetVariableValueData, GetVariableValueResponse, ExistsVariableData, ExistsVariableResponse, ListVariableData, ListVariableResponse, ListContextualVariablesData, ListContextualVariablesResponse, LoginWithOauthData, LoginWithOauthResponse, ConnectSlackCallbackData, ConnectSlackCallbackResponse, ConnectSlackCallbackInstanceData, ConnectSlackCallbackInstanceResponse, ConnectCallbackData, ConnectCallbackResponse, CreateAccountData, CreateAccountResponse, RefreshTokenData, RefreshTokenResponse, DisconnectAccountData, DisconnectAccountResponse, DisconnectSlackData, DisconnectSlackResponse, ListOauthLoginsResponse, ListOauthConnectsResponse, GetOauthConnectData, GetOauthConnectResponse, CreateResourceData, CreateResourceResponse, DeleteResourceData, DeleteResourceResponse, UpdateResourceData, UpdateResourceResponse, UpdateResourceValueData, UpdateResourceValueResponse, GetResourceData, GetResourceResponse, GetResourceValueInterpolatedData, GetResourceValueInterpolatedResponse, GetResourceValueData, GetResourceValueResponse, ExistsResourceData, ExistsResourceResponse, ListResourceData, ListResourceResponse, ListSearchResourceData, ListSearchResourceResponse, ListResourceNamesData, ListResourceNamesResponse, CreateResourceTypeData, CreateResourceTypeResponse, FileResourceTypeToFileExtMapData, FileResourceTypeToFileExtMapResponse, DeleteResourceTypeData, DeleteResourceTypeResponse, UpdateResourceTypeData, UpdateResourceTypeResponse, GetResourceTypeData, GetResourceTypeResponse, ExistsResourceTypeData, ExistsResourceTypeResponse, ListResourceTypeData, ListResourceTypeResponse, ListResourceTypeNamesData, ListResourceTypeNamesResponse, QueryResourceTypesData, QueryResourceTypesResponse, ListHubIntegrationsData, ListHubIntegrationsResponse, ListHubFlowsResponse, GetHubFlowByIdData, GetHubFlowByIdResponse, ListHubAppsResponse, GetHubAppByIdData, GetHubAppByIdResponse, GetHubScriptContentByPathData, GetHubScriptContentByPathResponse, GetHubScriptByPathData, GetHubScriptByPathResponse, GetTopHubScriptsData, GetTopHubScriptsResponse, QueryHubScriptsData, QueryHubScriptsResponse, ListSearchScriptData, ListSearchScriptResponse, ListScriptsData, ListScriptsResponse, ListScriptPathsData, ListScriptPathsResponse, CreateDraftData, CreateDraftResponse, DeleteDraftData, DeleteDraftResponse, CreateScriptData, CreateScriptResponse, ToggleWorkspaceErrorHandlerForScriptData, ToggleWorkspaceErrorHandlerForScriptResponse, GetCustomTagsResponse, GeDefaultTagsResponse, IsDefaultTagsPerWorkspaceResponse, ArchiveScriptByPathData, ArchiveScriptByPathResponse, ArchiveScriptByHashData, ArchiveScriptByHashResponse, DeleteScriptByHashData, DeleteScriptByHashResponse, DeleteScriptByPathData, DeleteScriptByPathResponse, GetScriptByPathData, GetScriptByPathResponse, GetScriptByPathWithDraftData, GetScriptByPathWithDraftResponse, GetScriptHistoryByPathData, GetScriptHistoryByPathResponse, UpdateScriptHistoryData, UpdateScriptHistoryResponse, RawScriptByPathData, RawScriptByPathResponse, RawScriptByPathTokenedData, RawScriptByPathTokenedResponse, ExistsScriptByPathData, ExistsScriptByPathResponse, GetScriptByHashData, GetScriptByHashResponse, RawScriptByHashData, RawScriptByHashResponse, GetScriptDeploymentStatusData, GetScriptDeploymentStatusResponse, RunScriptByPathData, RunScriptByPathResponse, OpenaiSyncScriptByPathData, OpenaiSyncScriptByPathResponse, RunWaitResultScriptByPathData, RunWaitResultScriptByPathResponse, RunWaitResultScriptByPathGetData, RunWaitResultScriptByPathGetResponse, OpenaiSyncFlowByPathData, OpenaiSyncFlowByPathResponse, RunWaitResultFlowByPathData, RunWaitResultFlowByPathResponse, ResultByIdData, ResultByIdResponse, ListFlowPathsData, ListFlowPathsResponse, ListSearchFlowData, ListSearchFlowResponse, ListFlowsData, ListFlowsResponse, GetFlowHistoryData, GetFlowHistoryResponse, GetFlowVersionData, GetFlowVersionResponse, UpdateFlowHistoryData, UpdateFlowHistoryResponse, GetFlowByPathData, GetFlowByPathResponse, ToggleWorkspaceErrorHandlerForFlowData, ToggleWorkspaceErrorHandlerForFlowResponse, GetFlowByPathWithDraftData, GetFlowByPathWithDraftResponse, ExistsFlowByPathData, ExistsFlowByPathResponse, CreateFlowData, CreateFlowResponse, UpdateFlowData, UpdateFlowResponse, ArchiveFlowByPathData, ArchiveFlowByPathResponse, DeleteFlowByPathData, DeleteFlowByPathResponse, ListRawAppsData, ListRawAppsResponse, ExistsRawAppData, ExistsRawAppResponse, GetRawAppDataData, GetRawAppDataResponse, ListSearchAppData, ListSearchAppResponse, ListAppsData, ListAppsResponse, CreateAppData, CreateAppResponse, ExistsAppData, ExistsAppResponse, GetAppByPathData, GetAppByPathResponse, GetAppByPathWithDraftData, GetAppByPathWithDraftResponse, GetAppHistoryByPathData, GetAppHistoryByPathResponse, UpdateAppHistoryData, UpdateAppHistoryResponse, GetPublicAppBySecretData, GetPublicAppBySecretResponse, GetPublicResourceData, GetPublicResourceResponse, GetPublicSecretOfAppData, GetPublicSecretOfAppResponse, GetAppByVersionData, GetAppByVersionResponse, CreateRawAppData, CreateRawAppResponse, UpdateRawAppData, UpdateRawAppResponse, DeleteRawAppData, DeleteRawAppResponse, DeleteAppData, DeleteAppResponse, UpdateAppData, UpdateAppResponse, ExecuteComponentData, ExecuteComponentResponse, RunFlowByPathData, RunFlowByPathResponse, RestartFlowAtStepData, RestartFlowAtStepResponse, RunScriptByHashData, RunScriptByHashResponse, RunScriptPreviewData, RunScriptPreviewResponse, RunCodeWorkflowTaskData, RunCodeWorkflowTaskResponse, RunRawScriptDependenciesData, RunRawScriptDependenciesResponse, RunFlowPreviewData, RunFlowPreviewResponse, ListQueueData, ListQueueResponse, GetQueueCountData, GetQueueCountResponse, GetCompletedCountData, GetCompletedCountResponse, ListFilteredUuidsData, ListFilteredUuidsResponse, CancelSelectionData, CancelSelectionResponse, ListCompletedJobsData, ListCompletedJobsResponse, ListJobsData, ListJobsResponse, GetDbClockResponse, GetJobData, GetJobResponse, GetRootJobIdData, GetRootJobIdResponse, GetJobLogsData, GetJobLogsResponse, GetJobArgsData, GetJobArgsResponse, GetJobUpdatesData, GetJobUpdatesResponse, GetLogFileFromStoreData, GetLogFileFromStoreResponse, GetFlowDebugInfoData, GetFlowDebugInfoResponse, GetCompletedJobData, GetCompletedJobResponse, GetCompletedJobResultData, GetCompletedJobResultResponse, GetCompletedJobResultMaybeData, GetCompletedJobResultMaybeResponse, DeleteCompletedJobData, DeleteCompletedJobResponse, CancelQueuedJobData, CancelQueuedJobResponse, CancelPersistentQueuedJobsData, CancelPersistentQueuedJobsResponse, ForceCancelQueuedJobData, ForceCancelQueuedJobResponse, CreateJobSignatureData, CreateJobSignatureResponse, GetResumeUrlsData, GetResumeUrlsResponse, ResumeSuspendedJobGetData, ResumeSuspendedJobGetResponse, ResumeSuspendedJobPostData, ResumeSuspendedJobPostResponse, SetFlowUserStateData, SetFlowUserStateResponse, GetFlowUserStateData, GetFlowUserStateResponse, ResumeSuspendedFlowAsOwnerData, ResumeSuspendedFlowAsOwnerResponse, CancelSuspendedJobGetData, CancelSuspendedJobGetResponse, CancelSuspendedJobPostData, CancelSuspendedJobPostResponse, GetSuspendedJobFlowData, GetSuspendedJobFlowResponse, PreviewScheduleData, PreviewScheduleResponse, CreateScheduleData, CreateScheduleResponse, UpdateScheduleData, UpdateScheduleResponse, SetScheduleEnabledData, SetScheduleEnabledResponse, DeleteScheduleData, DeleteScheduleResponse, GetScheduleData, GetScheduleResponse, ExistsScheduleData, ExistsScheduleResponse, ListSchedulesData, ListSchedulesResponse, ListSchedulesWithJobsData, ListSchedulesWithJobsResponse, SetDefaultErrorOrRecoveryHandlerData, SetDefaultErrorOrRecoveryHandlerResponse, CreateHttpTriggerData, CreateHttpTriggerResponse, UpdateHttpTriggerData, UpdateHttpTriggerResponse, DeleteHttpTriggerData, DeleteHttpTriggerResponse, GetHttpTriggerData, GetHttpTriggerResponse, ListHttpTriggersData, ListHttpTriggersResponse, ExistsHttpTriggerData, ExistsHttpTriggerResponse, ExistsRouteData, ExistsRouteResponse, UsedData, UsedResponse, ListInstanceGroupsResponse, GetInstanceGroupData, GetInstanceGroupResponse, CreateInstanceGroupData, CreateInstanceGroupResponse, UpdateInstanceGroupData, UpdateInstanceGroupResponse, DeleteInstanceGroupData, DeleteInstanceGroupResponse, AddUserToInstanceGroupData, AddUserToInstanceGroupResponse, RemoveUserFromInstanceGroupData, RemoveUserFromInstanceGroupResponse, ExportInstanceGroupsResponse, OverwriteInstanceGroupsData, OverwriteInstanceGroupsResponse, ListGroupsData, ListGroupsResponse, ListGroupNamesData, ListGroupNamesResponse, CreateGroupData, CreateGroupResponse, UpdateGroupData, UpdateGroupResponse, DeleteGroupData, DeleteGroupResponse, GetGroupData, GetGroupResponse, AddUserToGroupData, AddUserToGroupResponse, RemoveUserToGroupData, RemoveUserToGroupResponse, ListFoldersData, ListFoldersResponse, ListFolderNamesData, ListFolderNamesResponse, CreateFolderData, CreateFolderResponse, UpdateFolderData, UpdateFolderResponse, DeleteFolderData, DeleteFolderResponse, GetFolderData, GetFolderResponse, GetFolderUsageData, GetFolderUsageResponse, AddOwnerToFolderData, AddOwnerToFolderResponse, RemoveOwnerToFolderData, RemoveOwnerToFolderResponse, ListWorkersData, ListWorkersResponse, ExistsWorkerWithTagData, ExistsWorkerWithTagResponse, GetQueueMetricsResponse, ListWorkerGroupsResponse, GetConfigData, GetConfigResponse, UpdateConfigData, UpdateConfigResponse, DeleteConfigData, DeleteConfigResponse, ListConfigsResponse, GetGranularAclsData, GetGranularAclsResponse, AddGranularAclsData, AddGranularAclsResponse, RemoveGranularAclsData, RemoveGranularAclsResponse, UpdateCaptureData, UpdateCaptureResponse, CreateCaptureData, CreateCaptureResponse, GetCaptureData, GetCaptureResponse, StarData, StarResponse, UnstarData, UnstarResponse, GetInputHistoryData, GetInputHistoryResponse, GetArgsFromHistoryOrSavedInputData, GetArgsFromHistoryOrSavedInputResponse, ListInputsData, ListInputsResponse, CreateInputData, CreateInputResponse, UpdateInputData, UpdateInputResponse, DeleteInputData, DeleteInputResponse, DuckdbConnectionSettingsData, DuckdbConnectionSettingsResponse, DuckdbConnectionSettingsV2Data, DuckdbConnectionSettingsV2Response, PolarsConnectionSettingsData, PolarsConnectionSettingsResponse, PolarsConnectionSettingsV2Data, PolarsConnectionSettingsV2Response, S3ResourceInfoData, S3ResourceInfoResponse, DatasetStorageTestConnectionData, DatasetStorageTestConnectionResponse, ListStoredFilesData, ListStoredFilesResponse, LoadFileMetadataData, LoadFileMetadataResponse, LoadFilePreviewData, LoadFilePreviewResponse, LoadParquetPreviewData, LoadParquetPreviewResponse, LoadTableRowCountData, LoadTableRowCountResponse, LoadCsvPreviewData, LoadCsvPreviewResponse, DeleteS3FileData, DeleteS3FileResponse, MoveS3FileData, MoveS3FileResponse, FileUploadData, FileUploadResponse, FileDownloadData, FileDownloadResponse, FileDownloadParquetAsCsvData, FileDownloadParquetAsCsvResponse, GetJobMetricsData, GetJobMetricsResponse, SetJobProgressData, SetJobProgressResponse, GetJobProgressData, GetJobProgressResponse, ListLogFilesData, ListLogFilesResponse, GetLogFileData, GetLogFileResponse, ListConcurrencyGroupsResponse, DeleteConcurrencyGroupData, DeleteConcurrencyGroupResponse, GetConcurrencyKeyData, GetConcurrencyKeyResponse, ListExtendedJobsData, ListExtendedJobsResponse, SearchJobsIndexData, SearchJobsIndexResponse } from './types.gen.ts'; +import type { BackendVersionResponse, BackendUptodateResponse, GetLicenseIdResponse, GetOpenApiYamlResponse, GetAuditLogData, GetAuditLogResponse, ListAuditLogsData, ListAuditLogsResponse, LoginData, LoginResponse, LogoutResponse, GetUserData, GetUserResponse, UpdateUserData, UpdateUserResponse, IsOwnerOfPathData, IsOwnerOfPathResponse, SetPasswordData, SetPasswordResponse, CreateUserGloballyData, CreateUserGloballyResponse, GlobalUserUpdateData, GlobalUserUpdateResponse, GlobalUsernameInfoData, GlobalUsernameInfoResponse, GlobalUserRenameData, GlobalUserRenameResponse, GlobalUserDeleteData, GlobalUserDeleteResponse, GlobalUsersOverwriteData, GlobalUsersOverwriteResponse, GlobalUsersExportResponse, DeleteUserData, DeleteUserResponse, ListWorkspacesResponse, IsDomainAllowedResponse, ListUserWorkspacesResponse, ListWorkspacesAsSuperAdminData, ListWorkspacesAsSuperAdminResponse, CreateWorkspaceData, CreateWorkspaceResponse, ExistsWorkspaceData, ExistsWorkspaceResponse, ExistsUsernameData, ExistsUsernameResponse, GetGlobalData, GetGlobalResponse, SetGlobalData, SetGlobalResponse, GetLocalResponse, TestSmtpData, TestSmtpResponse, TestCriticalChannelsData, TestCriticalChannelsResponse, TestLicenseKeyData, TestLicenseKeyResponse, TestObjectStorageConfigData, TestObjectStorageConfigResponse, SendStatsResponse, GetLatestKeyRenewalAttemptResponse, RenewLicenseKeyData, RenewLicenseKeyResponse, CreateCustomerPortalSessionData, CreateCustomerPortalSessionResponse, TestMetadataData, TestMetadataResponse, ListGlobalSettingsResponse, GetCurrentEmailResponse, RefreshUserTokenResponse, GetTutorialProgressResponse, UpdateTutorialProgressData, UpdateTutorialProgressResponse, LeaveInstanceResponse, GetUsageResponse, GetRunnableResponse, GlobalWhoamiResponse, ListWorkspaceInvitesResponse, WhoamiData, WhoamiResponse, AcceptInviteData, AcceptInviteResponse, DeclineInviteData, DeclineInviteResponse, InviteUserData, InviteUserResponse, AddUserData, AddUserResponse, DeleteInviteData, DeleteInviteResponse, ArchiveWorkspaceData, ArchiveWorkspaceResponse, UnarchiveWorkspaceData, UnarchiveWorkspaceResponse, DeleteWorkspaceData, DeleteWorkspaceResponse, LeaveWorkspaceData, LeaveWorkspaceResponse, GetWorkspaceNameData, GetWorkspaceNameResponse, ChangeWorkspaceNameData, ChangeWorkspaceNameResponse, ChangeWorkspaceIdData, ChangeWorkspaceIdResponse, WhoisData, WhoisResponse, ExistsEmailData, ExistsEmailResponse, ListUsersAsSuperAdminData, ListUsersAsSuperAdminResponse, ListPendingInvitesData, ListPendingInvitesResponse, GetSettingsData, GetSettingsResponse, GetDeployToData, GetDeployToResponse, GetIsPremiumData, GetIsPremiumResponse, GetPremiumInfoData, GetPremiumInfoResponse, SetAutomaticBillingData, SetAutomaticBillingResponse, EditSlackCommandData, EditSlackCommandResponse, RunSlackMessageTestJobData, RunSlackMessageTestJobResponse, EditDeployToData, EditDeployToResponse, EditAutoInviteData, EditAutoInviteResponse, EditWebhookData, EditWebhookResponse, EditCopilotConfigData, EditCopilotConfigResponse, GetCopilotInfoData, GetCopilotInfoResponse, EditErrorHandlerData, EditErrorHandlerResponse, EditLargeFileStorageConfigData, EditLargeFileStorageConfigResponse, EditWorkspaceGitSyncConfigData, EditWorkspaceGitSyncConfigResponse, EditWorkspaceDeployUiSettingsData, EditWorkspaceDeployUiSettingsResponse, EditWorkspaceDefaultAppData, EditWorkspaceDefaultAppResponse, EditDefaultScriptsData, EditDefaultScriptsResponse, GetDefaultScriptsData, GetDefaultScriptsResponse, SetEnvironmentVariableData, SetEnvironmentVariableResponse, GetWorkspaceEncryptionKeyData, GetWorkspaceEncryptionKeyResponse, SetWorkspaceEncryptionKeyData, SetWorkspaceEncryptionKeyResponse, GetWorkspaceDefaultAppData, GetWorkspaceDefaultAppResponse, GetLargeFileStorageConfigData, GetLargeFileStorageConfigResponse, GetWorkspaceUsageData, GetWorkspaceUsageResponse, ListUsersData, ListUsersResponse, ListUsersUsageData, ListUsersUsageResponse, ListUsernamesData, ListUsernamesResponse, UsernameToEmailData, UsernameToEmailResponse, CreateTokenData, CreateTokenResponse, CreateTokenImpersonateData, CreateTokenImpersonateResponse, DeleteTokenData, DeleteTokenResponse, ListTokensData, ListTokensResponse, GetOidcTokenData, GetOidcTokenResponse, CreateVariableData, CreateVariableResponse, EncryptValueData, EncryptValueResponse, DeleteVariableData, DeleteVariableResponse, UpdateVariableData, UpdateVariableResponse, GetVariableData, GetVariableResponse, GetVariableValueData, GetVariableValueResponse, ExistsVariableData, ExistsVariableResponse, ListVariableData, ListVariableResponse, ListContextualVariablesData, ListContextualVariablesResponse, LoginWithOauthData, LoginWithOauthResponse, ConnectSlackCallbackData, ConnectSlackCallbackResponse, ConnectSlackCallbackInstanceData, ConnectSlackCallbackInstanceResponse, ConnectCallbackData, ConnectCallbackResponse, CreateAccountData, CreateAccountResponse, RefreshTokenData, RefreshTokenResponse, DisconnectAccountData, DisconnectAccountResponse, DisconnectSlackData, DisconnectSlackResponse, ListOauthLoginsResponse, ListOauthConnectsResponse, GetOauthConnectData, GetOauthConnectResponse, CreateResourceData, CreateResourceResponse, DeleteResourceData, DeleteResourceResponse, UpdateResourceData, UpdateResourceResponse, UpdateResourceValueData, UpdateResourceValueResponse, GetResourceData, GetResourceResponse, GetResourceValueInterpolatedData, GetResourceValueInterpolatedResponse, GetResourceValueData, GetResourceValueResponse, ExistsResourceData, ExistsResourceResponse, ListResourceData, ListResourceResponse, ListSearchResourceData, ListSearchResourceResponse, ListResourceNamesData, ListResourceNamesResponse, CreateResourceTypeData, CreateResourceTypeResponse, FileResourceTypeToFileExtMapData, FileResourceTypeToFileExtMapResponse, DeleteResourceTypeData, DeleteResourceTypeResponse, UpdateResourceTypeData, UpdateResourceTypeResponse, GetResourceTypeData, GetResourceTypeResponse, ExistsResourceTypeData, ExistsResourceTypeResponse, ListResourceTypeData, ListResourceTypeResponse, ListResourceTypeNamesData, ListResourceTypeNamesResponse, QueryResourceTypesData, QueryResourceTypesResponse, ListHubIntegrationsData, ListHubIntegrationsResponse, ListHubFlowsResponse, GetHubFlowByIdData, GetHubFlowByIdResponse, ListHubAppsResponse, GetHubAppByIdData, GetHubAppByIdResponse, GetHubScriptContentByPathData, GetHubScriptContentByPathResponse, GetHubScriptByPathData, GetHubScriptByPathResponse, GetTopHubScriptsData, GetTopHubScriptsResponse, QueryHubScriptsData, QueryHubScriptsResponse, ListSearchScriptData, ListSearchScriptResponse, ListScriptsData, ListScriptsResponse, ListScriptPathsData, ListScriptPathsResponse, CreateDraftData, CreateDraftResponse, DeleteDraftData, DeleteDraftResponse, CreateScriptData, CreateScriptResponse, ToggleWorkspaceErrorHandlerForScriptData, ToggleWorkspaceErrorHandlerForScriptResponse, GetCustomTagsResponse, GeDefaultTagsResponse, IsDefaultTagsPerWorkspaceResponse, ArchiveScriptByPathData, ArchiveScriptByPathResponse, ArchiveScriptByHashData, ArchiveScriptByHashResponse, DeleteScriptByHashData, DeleteScriptByHashResponse, DeleteScriptByPathData, DeleteScriptByPathResponse, GetScriptByPathData, GetScriptByPathResponse, GetScriptByPathWithDraftData, GetScriptByPathWithDraftResponse, GetScriptHistoryByPathData, GetScriptHistoryByPathResponse, UpdateScriptHistoryData, UpdateScriptHistoryResponse, RawScriptByPathData, RawScriptByPathResponse, RawScriptByPathTokenedData, RawScriptByPathTokenedResponse, ExistsScriptByPathData, ExistsScriptByPathResponse, GetScriptByHashData, GetScriptByHashResponse, RawScriptByHashData, RawScriptByHashResponse, GetScriptDeploymentStatusData, GetScriptDeploymentStatusResponse, RunScriptByPathData, RunScriptByPathResponse, OpenaiSyncScriptByPathData, OpenaiSyncScriptByPathResponse, RunWaitResultScriptByPathData, RunWaitResultScriptByPathResponse, RunWaitResultScriptByPathGetData, RunWaitResultScriptByPathGetResponse, OpenaiSyncFlowByPathData, OpenaiSyncFlowByPathResponse, RunWaitResultFlowByPathData, RunWaitResultFlowByPathResponse, ResultByIdData, ResultByIdResponse, ListFlowPathsData, ListFlowPathsResponse, ListSearchFlowData, ListSearchFlowResponse, ListFlowsData, ListFlowsResponse, GetFlowHistoryData, GetFlowHistoryResponse, GetFlowVersionData, GetFlowVersionResponse, UpdateFlowHistoryData, UpdateFlowHistoryResponse, GetFlowByPathData, GetFlowByPathResponse, ToggleWorkspaceErrorHandlerForFlowData, ToggleWorkspaceErrorHandlerForFlowResponse, GetFlowByPathWithDraftData, GetFlowByPathWithDraftResponse, ExistsFlowByPathData, ExistsFlowByPathResponse, CreateFlowData, CreateFlowResponse, UpdateFlowData, UpdateFlowResponse, ArchiveFlowByPathData, ArchiveFlowByPathResponse, DeleteFlowByPathData, DeleteFlowByPathResponse, ListRawAppsData, ListRawAppsResponse, ExistsRawAppData, ExistsRawAppResponse, GetRawAppDataData, GetRawAppDataResponse, ListSearchAppData, ListSearchAppResponse, ListAppsData, ListAppsResponse, CreateAppData, CreateAppResponse, ExistsAppData, ExistsAppResponse, GetAppByPathData, GetAppByPathResponse, GetAppByPathWithDraftData, GetAppByPathWithDraftResponse, GetAppHistoryByPathData, GetAppHistoryByPathResponse, UpdateAppHistoryData, UpdateAppHistoryResponse, GetPublicAppBySecretData, GetPublicAppBySecretResponse, GetPublicResourceData, GetPublicResourceResponse, GetPublicSecretOfAppData, GetPublicSecretOfAppResponse, GetAppByVersionData, GetAppByVersionResponse, CreateRawAppData, CreateRawAppResponse, UpdateRawAppData, UpdateRawAppResponse, DeleteRawAppData, DeleteRawAppResponse, DeleteAppData, DeleteAppResponse, UpdateAppData, UpdateAppResponse, ExecuteComponentData, ExecuteComponentResponse, RunFlowByPathData, RunFlowByPathResponse, RestartFlowAtStepData, RestartFlowAtStepResponse, RunScriptByHashData, RunScriptByHashResponse, RunScriptPreviewData, RunScriptPreviewResponse, RunCodeWorkflowTaskData, RunCodeWorkflowTaskResponse, RunRawScriptDependenciesData, RunRawScriptDependenciesResponse, RunFlowPreviewData, RunFlowPreviewResponse, ListQueueData, ListQueueResponse, GetQueueCountData, GetQueueCountResponse, GetCompletedCountData, GetCompletedCountResponse, ListFilteredUuidsData, ListFilteredUuidsResponse, CancelSelectionData, CancelSelectionResponse, ListCompletedJobsData, ListCompletedJobsResponse, ListJobsData, ListJobsResponse, GetDbClockResponse, CountJobsByTagData, CountJobsByTagResponse, GetJobData, GetJobResponse, GetRootJobIdData, GetRootJobIdResponse, GetJobLogsData, GetJobLogsResponse, GetJobArgsData, GetJobArgsResponse, GetJobUpdatesData, GetJobUpdatesResponse, GetLogFileFromStoreData, GetLogFileFromStoreResponse, GetFlowDebugInfoData, GetFlowDebugInfoResponse, GetCompletedJobData, GetCompletedJobResponse, GetCompletedJobResultData, GetCompletedJobResultResponse, GetCompletedJobResultMaybeData, GetCompletedJobResultMaybeResponse, DeleteCompletedJobData, DeleteCompletedJobResponse, CancelQueuedJobData, CancelQueuedJobResponse, CancelPersistentQueuedJobsData, CancelPersistentQueuedJobsResponse, ForceCancelQueuedJobData, ForceCancelQueuedJobResponse, CreateJobSignatureData, CreateJobSignatureResponse, GetResumeUrlsData, GetResumeUrlsResponse, ResumeSuspendedJobGetData, ResumeSuspendedJobGetResponse, ResumeSuspendedJobPostData, ResumeSuspendedJobPostResponse, SetFlowUserStateData, SetFlowUserStateResponse, GetFlowUserStateData, GetFlowUserStateResponse, ResumeSuspendedFlowAsOwnerData, ResumeSuspendedFlowAsOwnerResponse, CancelSuspendedJobGetData, CancelSuspendedJobGetResponse, CancelSuspendedJobPostData, CancelSuspendedJobPostResponse, GetSuspendedJobFlowData, GetSuspendedJobFlowResponse, PreviewScheduleData, PreviewScheduleResponse, CreateScheduleData, CreateScheduleResponse, UpdateScheduleData, UpdateScheduleResponse, SetScheduleEnabledData, SetScheduleEnabledResponse, DeleteScheduleData, DeleteScheduleResponse, GetScheduleData, GetScheduleResponse, ExistsScheduleData, ExistsScheduleResponse, ListSchedulesData, ListSchedulesResponse, ListSchedulesWithJobsData, ListSchedulesWithJobsResponse, SetDefaultErrorOrRecoveryHandlerData, SetDefaultErrorOrRecoveryHandlerResponse, CreateHttpTriggerData, CreateHttpTriggerResponse, UpdateHttpTriggerData, UpdateHttpTriggerResponse, DeleteHttpTriggerData, DeleteHttpTriggerResponse, GetHttpTriggerData, GetHttpTriggerResponse, ListHttpTriggersData, ListHttpTriggersResponse, ExistsHttpTriggerData, ExistsHttpTriggerResponse, ExistsRouteData, ExistsRouteResponse, UsedData, UsedResponse, ListInstanceGroupsResponse, GetInstanceGroupData, GetInstanceGroupResponse, CreateInstanceGroupData, CreateInstanceGroupResponse, UpdateInstanceGroupData, UpdateInstanceGroupResponse, DeleteInstanceGroupData, DeleteInstanceGroupResponse, AddUserToInstanceGroupData, AddUserToInstanceGroupResponse, RemoveUserFromInstanceGroupData, RemoveUserFromInstanceGroupResponse, ExportInstanceGroupsResponse, OverwriteInstanceGroupsData, OverwriteInstanceGroupsResponse, ListGroupsData, ListGroupsResponse, ListGroupNamesData, ListGroupNamesResponse, CreateGroupData, CreateGroupResponse, UpdateGroupData, UpdateGroupResponse, DeleteGroupData, DeleteGroupResponse, GetGroupData, GetGroupResponse, AddUserToGroupData, AddUserToGroupResponse, RemoveUserToGroupData, RemoveUserToGroupResponse, ListFoldersData, ListFoldersResponse, ListFolderNamesData, ListFolderNamesResponse, CreateFolderData, CreateFolderResponse, UpdateFolderData, UpdateFolderResponse, DeleteFolderData, DeleteFolderResponse, GetFolderData, GetFolderResponse, GetFolderUsageData, GetFolderUsageResponse, AddOwnerToFolderData, AddOwnerToFolderResponse, RemoveOwnerToFolderData, RemoveOwnerToFolderResponse, ListWorkersData, ListWorkersResponse, ExistsWorkerWithTagData, ExistsWorkerWithTagResponse, GetQueueMetricsResponse, ListWorkerGroupsResponse, GetConfigData, GetConfigResponse, UpdateConfigData, UpdateConfigResponse, DeleteConfigData, DeleteConfigResponse, ListConfigsResponse, GetGranularAclsData, GetGranularAclsResponse, AddGranularAclsData, AddGranularAclsResponse, RemoveGranularAclsData, RemoveGranularAclsResponse, UpdateCaptureData, UpdateCaptureResponse, CreateCaptureData, CreateCaptureResponse, GetCaptureData, GetCaptureResponse, StarData, StarResponse, UnstarData, UnstarResponse, GetInputHistoryData, GetInputHistoryResponse, GetArgsFromHistoryOrSavedInputData, GetArgsFromHistoryOrSavedInputResponse, ListInputsData, ListInputsResponse, CreateInputData, CreateInputResponse, UpdateInputData, UpdateInputResponse, DeleteInputData, DeleteInputResponse, DuckdbConnectionSettingsData, DuckdbConnectionSettingsResponse, DuckdbConnectionSettingsV2Data, DuckdbConnectionSettingsV2Response, PolarsConnectionSettingsData, PolarsConnectionSettingsResponse, PolarsConnectionSettingsV2Data, PolarsConnectionSettingsV2Response, S3ResourceInfoData, S3ResourceInfoResponse, DatasetStorageTestConnectionData, DatasetStorageTestConnectionResponse, ListStoredFilesData, ListStoredFilesResponse, LoadFileMetadataData, LoadFileMetadataResponse, LoadFilePreviewData, LoadFilePreviewResponse, LoadParquetPreviewData, LoadParquetPreviewResponse, LoadTableRowCountData, LoadTableRowCountResponse, LoadCsvPreviewData, LoadCsvPreviewResponse, DeleteS3FileData, DeleteS3FileResponse, MoveS3FileData, MoveS3FileResponse, FileUploadData, FileUploadResponse, FileDownloadData, FileDownloadResponse, FileDownloadParquetAsCsvData, FileDownloadParquetAsCsvResponse, GetJobMetricsData, GetJobMetricsResponse, SetJobProgressData, SetJobProgressResponse, GetJobProgressData, GetJobProgressResponse, ListLogFilesData, ListLogFilesResponse, GetLogFileData, GetLogFileResponse, ListConcurrencyGroupsResponse, DeleteConcurrencyGroupData, DeleteConcurrencyGroupResponse, GetConcurrencyKeyData, GetConcurrencyKeyResponse, ListExtendedJobsData, ListExtendedJobsResponse, SearchJobsIndexData, SearchJobsIndexResponse } from './types.gen.ts'; /** * get backend version @@ -4284,6 +4284,23 @@ export const getDbClock = (): CancelablePromise => { return url: '/jobs/db_clock' }); }; +/** + * Count jobs by tag + * @param data The data for the request. + * @param data.horizonSecs Past Time horizon in seconds (when to start the count = now - horizon) (default is 3600) + * @param data.workspaceId Specific workspace ID to filter results (optional) + * @returns unknown Job counts by tag + * @throws ApiError + */ +export const countJobsByTag = (data: CountJobsByTagData = {}): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/jobs/completed/count_by_tag', + query: { + horizon_secs: data.horizonSecs, + workspace_id: data.workspaceId + } +}); }; + /** * get job * @param data The data for the request. diff --git a/cli/gen/types.gen.ts b/cli/gen/types.gen.ts index e066cf1436..92bfe933d9 100644 --- a/cli/gen/types.gen.ts +++ b/cli/gen/types.gen.ts @@ -160,6 +160,7 @@ export type QueuedJob = { priority?: number; self_wait_time_ms?: number; aggregate_wait_time_ms?: number; + suspend?: number; }; export type job_kind = 'script' | 'preview' | 'dependencies' | 'flowdependencies' | 'appdependencies' | 'flow' | 'flowpreview' | 'script_hub' | 'identity' | 'deploymentcallback' | 'singlescriptflow'; @@ -613,6 +614,9 @@ export type WorkerPing = { last_job_id?: string; last_job_workspace_id?: string; occupancy_rate?: number; + occupancy_rate_15s?: number; + occupancy_rate_5m?: number; + occupancy_rate_30m?: number; memory?: number; vcpus?: number; memory_usage?: number; @@ -4406,6 +4410,22 @@ export type ListJobsResponse = (Array); export type GetDbClockResponse = (number); +export type CountJobsByTagData = { + /** + * Past Time horizon in seconds (when to start the count = now - horizon) (default is 3600) + */ + horizonSecs?: number; + /** + * Specific workspace ID to filter results (optional) + */ + workspaceId?: string; +}; + +export type CountJobsByTagResponse = (Array<{ + tag: string; + count: number; +}>); + export type GetJobData = { id: string; noLogs?: boolean; diff --git a/cli/instance.ts b/cli/instance.ts index e71f237d52..cf2d5f5b02 100644 --- a/cli/instance.ts +++ b/cli/instance.ts @@ -6,6 +6,7 @@ import { yamlParse, Command, setClient, + Table, } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; @@ -62,39 +63,49 @@ export async function allInstances(): Promise { return []; } } -export async function addInstance() { - let remote = await Input.prompt({ - message: "Enter the remote url of this instance", - default: "https://app.windmill.dev/", - }); - remote = new URL(remote).toString(); // add trailing slash in all cases! +export async function addInstance( + opts: {}, + instanceName: string | undefined, + remote: string | undefined, + token: string | undefined +) { + if (!remote) { + remote = await Input.prompt({ + message: "Enter the remote url of this instance", + default: "https://my.windmill.dev/", + }); + remote = new URL(remote).toString(); // add trailing slash in all cases! + } - const defaultName = new URL(remote).hostname; + if (!instanceName) { + const defaultName = new URL(remote).hostname.split(".")[0]; - const name = await Input.prompt({ - message: "Enter a name for this instance", - default: defaultName, - }); + instanceName = await Input.prompt({ + message: "Enter a name for this instance", + default: defaultName, + }); + } + const prefix = instanceName.toLowerCase().replace(/[^a-z0-9]/g, ""); - const prefix = name.toLowerCase().replace(/[^a-z0-9]/g, ""); - - let token: string | undefined = undefined; while (!token) { token = await loginInteractive(remote); } await appendInstance({ - name, + name: instanceName, remote, token, prefix, }); log.info( - colors.green.underline(`Added instance ${name} with remote ${remote}!`) + colors.green.underline( + `Added instance ${instanceName} with remote ${remote}!` + ) ); + await switchI({}, instanceName); return { - name, + name: instanceName, remote, token, prefix, @@ -160,37 +171,59 @@ export function compareInstanceObjects( return changes; } -type InstanceSyncOptions = { +export type InstanceSyncOptions = { skipUsers?: boolean; skipSettings?: boolean; skipConfigs?: boolean; skipGroups?: boolean; includeWorkspaces?: boolean; + instance?: string; baseUrl?: string; + token?: string; + yes?: boolean; }; -async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { +export async function pickInstance(opts: InstanceSyncOptions, allowNew: boolean) { const instances = await allInstances(); - let instance: Instance; - if (instances.length < 1) { - instance = await addInstance(); - } else { - const choice = (await Select.prompt({ - message: "Select an instance to pull from", - options: [ - ...instances.map((i) => ({ - name: `${i.name} (${i.remote})`, - value: i.name, - })), - { name: "Add new instance", value: "new" }, - ], - })) as unknown as string; - - if (choice === "new") { - instance = await addInstance(); + if (opts.baseUrl && opts.token) { + log.info("Using instance fully defined by --base-url and --token") + return { + name: "custom", + remote: opts.baseUrl, + token: opts.token, + prefix: "custom", + }; + } + if (!allowNew && instances.length < 1) { + throw new Error("No instance found, please add one first"); + } + const instanceName = await getActiveInstance(opts); + let instance: Instance | undefined = instances.find( + (i) => i.name === instanceName + ); + if (!instance) { + if (instances.length < 1) { + instance = await addInstance({}, undefined, undefined, undefined); } else { - instance = instances.find((i) => i.name === choice)!; + const choice = (await Select.prompt({ + message: "Select an instance", + options: [ + ...instances.map((i) => ({ + name: `${i.name} (${i.remote})`, + value: i.name, + })), + { name: "Add new instance", value: "new" }, + ], + })) as unknown as string; + + if (choice === "new") { + instance = await addInstance({}, undefined, undefined, undefined); + } else { + instance = instances.find((i) => i.name === choice)!; + } } + } else { + log.info(`Selected instance: ${instance.name}`); } setClient( @@ -198,6 +231,10 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { instance.remote.slice(0, instance.remote.length - 1) ); + return instance; +} +async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { + const instance = await pickInstance(opts, true); log.info("Pulling instance-level changes"); log.info(`remote (${instance.name}) -> local`); @@ -221,10 +258,13 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { const totalChanges = uChanges + sChanges + cChanges + gChanges; if (totalChanges > 0) { - const confirm = await Confirm.prompt({ - message: `Do you want to apply these ${totalChanges} instance-level changes?`, - default: true, - }); + let confirm = true; + if (opts.yes !== true) { + confirm = await Confirm.prompt({ + message: `Do you want to pull these ${totalChanges} instance-level changes?`, + default: true, + }); + } if (confirm) { if (!opts.skipUsers && uChanges > 0) { @@ -318,32 +358,7 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { let instances = await allInstances(); - let instance: Instance; - if (instances.length < 1) { - instance = await addInstance(); - } else { - const choice = (await Select.prompt({ - message: "Select an instance to push to", - options: [ - ...instances.map((i) => ({ - name: `${i.name} (${i.remote})`, - value: i.name, - })), - { name: "Add new instance", value: "new" }, - ], - })) as unknown as string; - - if (choice === "new") { - instance = await addInstance(); - } else { - instance = instances.find((i) => i.name === choice)!; - } - } - - setClient( - instance.token, - instance.remote.slice(0, instance.remote.length - 1) - ); + const instance = await pickInstance(opts, true); log.info("Pushing instance-level changes"); log.info!(`remote (${instance.name}) <- local`); @@ -368,10 +383,13 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { const totalChanges = uChanges + sChanges + cChanges + gChanges; if (totalChanges > 0) { - const confirm = await Confirm.prompt({ - message: `Do you want to apply these ${totalChanges} instance-level changes?`, - default: true, - }); + let confirm = true; + if (opts.yes !== true) { + confirm = await Confirm.prompt({ + message: `Do you want to apply these ${totalChanges} instance-level changes?`, + default: true, + }); + } if (confirm) { if (!opts.skipUsers && uChanges > 0) { @@ -485,36 +503,140 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { } } +async function switchI(opts: {}, instanceName: string) { + const all = await allInstances(); + if (all.findIndex((x) => x.name === instanceName) === -1) { + log.info( + colors.red.bold(`! This instance ${instanceName} does not exist locally.`) + ); + log.info("available instances:"); + for (const w of all) { + log.info(" - " + w.name); + } + return; + } + + await Deno.writeTextFile( + (await getRootStore()) + "/activeInstance", + instanceName + ); + + log.info(colors.green.underline(`Switched to instance ${instanceName}`)); +} + +export async function getActiveInstance(opts: { + instance?: string; +}): Promise { + if (opts.instance) { + return opts.instance; + } + try { + return await Deno.readTextFile((await getRootStore()) + "/activeInstance"); + } catch { + return undefined; + } +} + +async function whoami(opts: {}) { + await pickInstance({}, false); + try { + const whoamiInfo = await wmill.globalWhoami(); + log.info(colors.green.underline(`global whoami infos:`)); + log.info(JSON.stringify(whoamiInfo, null, 2)); + } catch (error) { + log.error(colors.red(`Failed to retrieve whoami information: ${error.message}`)); + } +} + const command = new Command() .description( "sync local with a remote instance or the opposite (push or pull)" ) - .action(() => - log.info("2 actions available, pull and push. Use -h to display help.") - ) + .action(async () => { + log.info( + "4 actions available, add, remove, switch, pull and push. Use -h to display help." + ); + const activeInstance = await getActiveInstance({}); + + new Table() + .header(["name", "remote", "token"]) + .padding(2) + .border(true) + .body( + (await allInstances()).map((x) => [ + x.name === activeInstance ? colors.underline(x.name) : x.name, + x.remote, + x.token.substring(0, 7) + "***", + ]) + ) + .render(); + if (activeInstance) { + log.info(`Selected instance: ${activeInstance}`); + } else { + log.info("No active instance selected"); + } + log.info("Use 'wmill instance add' to add a new instance"); + }) + .command("add") + .description("Add a new instance") + .action(addInstance as any) + .arguments("[instance_name:string] [remote:string] [token:string]") + .command("remove") + .description("Remove an instance") + .complete("instance", async () => (await allInstances()).map((x) => x.name)) + .arguments("") + .action(async (instance) => { + const instances = await allInstances(); + + const choice = (await Select.prompt({ + message: "Select an instance to remove", + options: instances.map((i) => ({ + name: `${i.name} (${i.remote})`, + value: i.name, + })), + })) as unknown as string; + + await removeInstance(choice); + log.info(colors.green.underline(`Removed instance ${choice}`)); + }) + .command("switch") + .complete("instance", async () => (await allInstances()).map((x) => x.name)) + .arguments("") + .description("Switch the current instance") + .action(switchI as any) .command("pull") .description( "Pull instance settings, users, configs, instance groups and overwrite local" ) + .option("--yes", "Pull without needing confirmation") .option("--skip-users", "Skip pulling users") .option("--skip-settings", "Skip pulling settings") .option("--skip-configs", "Skip pulling configs (worker groups and SMTP)") .option("--skip-groups", "Skip pulling instance groups") .option("--include-workspaces", "Also pull workspaces") + .action(instancePull as any) .command("push") .description( "Push instance settings, users, configs, group and overwrite remote" ) + .option("--yes", "Push without needing confirmation") .option("--skip-users", "Skip pushing users") .option("--skip-settings", "Skip pushing settings") .option("--skip-configs", "Skip pushing configs (worker groups and SMTP)") .option("--skip-groups", "Skip pushing instance groups") .option("--include-workspaces", "Also push workspaces") .option( - "--base-url", - "Base url to be passed to the instance settings instead of the local one" + "--instance", + "Name of the instance to push to, override the active instance" ) - .action(instancePush as any); + .option( + "--base-url", + "If used with --token, will be used as the base url for the instance" + ) + .action(instancePush as any) + .command("whoami") + .description("Display information about the currently logged-in user") + .action(whoami as any); export default command; diff --git a/cli/main.ts b/cli/main.ts index b5f9bb0899..c1aa86cf3e 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -19,6 +19,8 @@ import folder from "./folder.ts"; import schedule from "./schedule.ts"; import sync from "./sync.ts"; import instance from "./instance.ts"; +import workerGroups from "./worker_groups.ts"; + import dev from "./dev.ts"; import { fetchVersion } from "./context.ts"; import { GlobalOptions } from "./types.ts"; @@ -28,6 +30,8 @@ import { NpmProvider } from "./upgrade.ts"; import { pull as hubPull } from "./hub.ts"; import { pull, push } from "./sync.ts"; import { add as workspaceAdd } from "./workspace.ts"; +import workers from "./workers.ts"; +import queues from "./queues.ts"; export { flow, @@ -56,14 +60,14 @@ export { // } // }); -export const VERSION = "1.401.0"; +export const VERSION = "1.402.3"; const command = new Command() .name("wmill") .action(() => log.info(`Welcome to Windmill CLI ${VERSION}. Use -h for help.`) ) - .description("A simple CLI tool for windmill.") + .description("Windmill CLI") .globalOption( "--workspace ", @@ -117,7 +121,9 @@ const command = new Command() .command("dev", dev) .command("sync", sync) .command("instance", instance) - + .command("worker-groups", workerGroups) + .command("workers", workers) + .command("queues", queues) .command("version", "Show version information") .action(async (opts) => { console.log("CLI build against " + VERSION); diff --git a/cli/queues.ts b/cli/queues.ts new file mode 100644 index 0000000000..12586f1abb --- /dev/null +++ b/cli/queues.ts @@ -0,0 +1,147 @@ +import { Command, Table } from "./deps.ts"; +import { log } from "./deps.ts"; +import * as wmill from "./gen/services.gen.ts"; +import { pickInstance } from "./instance.ts"; + +type Data = { + count: number; + later: number; + suspended: number; + waiting: number; + running: number; + rps30s: string; + rps5min: string; + rps30min: string; + rps24h: string; +} + +type GlobalOptions = { + instance?: string; + baseUrl?: string; +}; + + +function createRow(tag: string, data: Record) { + if (data[tag]) { + return; + } else { + data[tag] = { + count: 0, + waiting: 0, + later: 0, + running: 0, + suspended: 0, + rps30s: "", + rps5min: "", + rps30min: "", + rps24h: "", + } + } +} +async function displayQueues(opts: GlobalOptions, workspace?: string) { + const activeInstance = await pickInstance(opts, true); + if (activeInstance) { + try { + const queuedJobs = await wmill.listQueue({workspace: workspace ?? 'admins', allWorkspaces: workspace === undefined, perPage: 100000}); + const jobCounts30s = await wmill.countJobsByTag({ + horizonSecs: 30, + workspaceId: workspace, + }); + const nowFromDb = new Date(await wmill.getDbClock()); + const jobCounts5min = await wmill.countJobsByTag({ + horizonSecs: 300, + workspaceId: workspace, + + }); + const jobCounts30min = await wmill.countJobsByTag({ + horizonSecs: 1800, + workspaceId: workspace, + + }); + const jobCounts24h = await wmill.countJobsByTag({ + horizonSecs: 86400, + workspaceId: workspace, + + }); + + const data: Record = {} + + for (const job of queuedJobs) { + createRow(job.tag, data); + const scheduledFor = new Date(job.scheduled_for ?? ""); + if (job.running) { + if (job.suspend) { + data[job.tag].suspended += 1; + } else { + data[job.tag].running += 1; + } + } else if (scheduledFor <= nowFromDb) { + data[job.tag].waiting += 1; + } else { + data[job.tag].later += 1; + } + } + + for (const count of jobCounts30s) { + const tag = count.tag; + createRow(tag, data); + data[tag].rps30s = (count.count / 30).toFixed(3); + } + for (const count of jobCounts5min) { + const tag = count.tag; + createRow(tag, data); + data[tag].rps5min = (count.count / 300).toFixed(3); + } + for (const count of jobCounts30min) { + const tag = count.tag; + createRow(tag, data); + data[tag].rps30min = (count.count / 1800).toFixed(3); + } + + for (const count of jobCounts24h) { + const tag = count.tag; + createRow(tag, data); + data[tag].rps24h = (count.count / 86400).toFixed(3); + } + + const table = new Table(); + table.header([ + "", + "Running", + "Waiting", + "Later", + "Suspended", + "RPS (30s)", + "RPS (5min)", + "RPS (30min)", + "RPS (24h)", + ]); + let body = [] + for (const tag in data) { + body.push([tag, data[tag].running, data[tag].waiting, data[tag].later, data[tag].suspended, data[tag].rps30s, data[tag].rps5min, data[tag].rps30min, data[tag].rps24h]); + } + table.body(body).render(); + + } catch (error) { + log.error("Failed to fetch queue metrics:", error); + } + } else { + log.info("No active instance found"); + log.info("Use 'wmill instance add' to add a new instance"); + } +} + +const command = new Command() + .description("List all queues with their metrics") + .arguments("[workspace:string] the optional workspace to filter by (default to all workspaces)") + .option( + "--instance [instance]", + "Name of the instance to push to, override the active instance" + ) + .option( + "--base-url [baseUrl]", + "If used with --token, will be used as the base url for the instance" + ) + .action(displayQueues as any); + +export default command; diff --git a/cli/settings.ts b/cli/settings.ts index 9ef51e80d2..b3a1b6a272 100644 --- a/cli/settings.ts +++ b/cli/settings.ts @@ -8,6 +8,7 @@ import { isSuperset } from "./types.ts"; import { deepEqual } from "./utils.ts"; import * as wmill from "./gen/services.gen.ts"; import { Config, GlobalSetting } from "./gen/types.gen.ts"; +import { removeWorkerPrefix } from "./worker_groups.ts"; export interface SimplifiedSettings { // slack_team_id?: string; @@ -354,7 +355,12 @@ export async function pushInstanceSettings( } export async function pullInstanceConfigs(preview = false) { - const remoteConfigs = await wmill.listConfigs(); + const remoteConfigs = (await wmill.listConfigs()).map((x) => { + return { + ...x, + name: removeWorkerPrefix(x.name), + }; + }); if (preview) { let localConfigs: Config[] = []; @@ -383,7 +389,12 @@ export async function pullInstanceConfigs(preview = false) { } export async function pushInstanceConfigs(preview: boolean = false) { - const remoteConfigs = await wmill.listConfigs(); + const remoteConfigs = (await wmill.listConfigs()).map((x) => { + return { + ...x, + name: removeWorkerPrefix(x.name), + }; + }); const localConfigs = (await Deno.readTextFile("instance_configs.yaml") .then((raw) => yamlParse(raw)) .catch(() => [])) as Config[]; @@ -404,7 +415,7 @@ export async function pushInstanceConfigs(preview: boolean = false) { } try { await wmill.updateConfig({ - name: config.name, + name: config.name.startsWith('worker__') ? config.name : `worker__${config.name}`, requestBody: config.config, }); } catch (err) { diff --git a/cli/worker_groups.ts b/cli/worker_groups.ts new file mode 100644 index 0000000000..bd1eb6f56f --- /dev/null +++ b/cli/worker_groups.ts @@ -0,0 +1,138 @@ +import { Command, Confirm, setClient, Table } from "./deps.ts"; + +import { log } from "./deps.ts"; +import { allInstances, getActiveInstance, Instance, InstanceSyncOptions, pickInstance } from "./instance.ts"; +import * as wmill from "./gen/services.gen.ts"; +import { pullInstanceConfigs, pushInstanceConfigs } from "./settings.ts"; + +type GlobalOptions = { + instance?: string; + baseUrl?: string; +}; +export async function getInstance(opts: GlobalOptions) { + const instances = await allInstances(); + + const instanceName = await getActiveInstance(opts); + const instance = instances.find((i) => i.name === instanceName); + if (instance) { + setClient( + instance.token, + instance.remote.slice(0, instance.remote.length - 1) + ); + } + return instance; +} + +export function removeWorkerPrefix(name: string) { + if (name.startsWith("worker__")) { + return name.substring(8); + } + return name; +} + +export async function displayWorkerGroups(opts: void) { + log.info("2 actions available, pull and push."); + const activeInstance = await getActiveInstance({}); + + if (activeInstance) { + log.info("Active instance: " + activeInstance); + const instance = await getInstance({}); + if (instance) { + const wGroups = await wmill.listWorkerGroups(); + new Table() + .header(["name", "config"]) + .padding(2) + .border(true) + .body(wGroups.map((x) => [removeWorkerPrefix(x.name), JSON.stringify(x.config, null, 2)])) + .render(); + } else { + log.error(`Instance ${activeInstance} not found`); + } + } else { + log.info("No active instance found"); + log.info("Use 'wmill instance add' to add a new instance"); + } +} +async function pullWorkerGroups(opts: InstanceSyncOptions) { + await pickInstance(opts, true); + + const totalChanges = await pullInstanceConfigs(true) ?? 0; + + if (totalChanges === 0) { + log.info("No changes to apply"); + return; + } + + let confirm = true; + if (opts.yes !== true) { + confirm = await Confirm.prompt({ + message: `Do you want to pul these ${totalChanges} instance-level changes?`, + default: true, + }); + } + + if (confirm) { + await pullInstanceConfigs(false); + } +} + +async function pushWorkerGroups(opts: InstanceSyncOptions) { + await pickInstance(opts, true); + + const totalChanges = await pushInstanceConfigs(true) ?? 0; + + if (totalChanges === 0) { + log.info("No changes to apply"); + return; + } + + let confirm = true; + if (opts.yes !== true) { + confirm = await Confirm.prompt({ + message: `Do you want to apply these ${totalChanges} instance-level changes?`, + default: true, + }); + } + + if (confirm) { + await pushInstanceConfigs(false); + } +} + + + +const command = new Command() + .description("display worker groups, pull and push worker groups configs") + .action(displayWorkerGroups) + .command("pull") + .description( + "Pull worker groups (similar to `wmill instance pull --skip-users --skip-settings --skip-groups`)" + ) + .option( + "--instance", + "Name of the instance to push to, override the active instance" + ) + .option( + "--base-url", + "Base url to be passed to the instance settings instead of the local one" + ) + .option("--yes", "Pull without needing confirmation") + .action(pullWorkerGroups as any) + .command("push") + .description( + "Push instance settings, users, configs, group and overwrite remote" + ) + .option( + "--instance [instance]", + "Name of the instance to push to, override the active instance" + ) + .option( + "--base-url [baseUrl]", + "If used with --token, will be used as the base url for the instance" + ) + .option("--yes", "Push without needing confirmation") + .action(pushWorkerGroups as any); + + + +export default command; diff --git a/cli/workers.ts b/cli/workers.ts new file mode 100644 index 0000000000..ff28a1c504 --- /dev/null +++ b/cli/workers.ts @@ -0,0 +1,105 @@ +import { Command, Table } from "./deps.ts"; +import { log } from "./deps.ts"; +import * as wmill from "./gen/services.gen.ts"; +import { pickInstance } from "./instance.ts"; + +type GlobalOptions = { + instance?: string; + baseUrl?: string; +}; + + +function toPercent(value: number | undefined): string { + return value != undefined ? `${(value * 100).toFixed(1)}%` : '?%'; +} + +async function displayWorkers(opts: GlobalOptions) { + const activeInstance = await pickInstance(opts, true); + + if (activeInstance) { + const workerGroups = await wmill.listWorkerGroups(); + const workers = await wmill.listWorkers({ + pingSince: 10 + }); + + const groupedWorkers = workerGroups.map(group => { + + + const groupWorkers = workers.filter(worker => worker.worker_group === group.name); + return { + groupName: group.name, + workers: groupWorkers + }; + }); + + // Add workers that don't belong to any worker group + const ungroupedWorkers = workers.filter(worker => + !workerGroups.some(group => group.name === worker.worker_group) + ); + + if (ungroupedWorkers.length > 0) { + ungroupedWorkers.forEach(worker => { + const groupName = worker.worker_group || "Ungrouped"; + let group = groupedWorkers.find(g => g.groupName === groupName); + if (!group) { + group = { groupName, workers: [] }; + groupedWorkers.push(group); + } + group.workers.push(worker); + }); + } + + // Sort groupedWorkers + groupedWorkers.sort((a, b) => { + // Always put 'default' first + if (a.groupName === 'default') return -1; + if (b.groupName === 'default') return 1; + + // Then sort by number of workers (descending order) + return b.workers.length - a.workers.length; + }); + + groupedWorkers.forEach(group => { + log.info(`\nWorker Group: ${group.groupName} (${group.workers.length} workers)`); + if (group.workers.length === 0) { + log.info(" No workers in this group"); + } else { + + new Table() + .header(["Worker ID", "Host", "Queues", "Jobs", "Occupancy rate 15s/5m/30m/ever)", "Last job", "Last Ping"]) + .padding(2) + .border(true) + .maxColWidth(30) + .body(group.workers.map(worker => [ + worker.worker, + worker.worker_instance, + worker.custom_tags?.join(', ') || '', + worker.jobs_executed, + `${toPercent(worker.occupancy_rate_15s)}/${toPercent(worker.occupancy_rate_5m)}/${toPercent(worker.occupancy_rate_30m)}/${toPercent(worker.occupancy_rate)}`, + + worker.last_job_id ? worker.last_job_id + ' ' +worker.last_job_workspace_id : '', + `${worker.last_ping}s ago` + ])) + .render(); + } + }); + + } else { + log.info("No active instance found"); + log.info("Use 'wmill instance add' to add a new instance"); + } +} + +const command = new Command() + .description("List all workers grouped by worker groups") + .option( + "--instance [instance]", + "Name of the instance to push to, override the active instance" + ) + .option( + "--base-url [baseUrl]", + "If used with --token, will be used as the base url for the instance" + ) + .action(displayWorkers as any); + +export default command; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8089c9a262..fbfe82be4e 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.401.0", + "version": "1.402.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.401.0", + "version": "1.402.3", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", diff --git a/frontend/package.json b/frontend/package.json index 05bf5e7c13..b85fc1caba 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.401.0", + "version": "1.402.3", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/DateInput.svelte b/frontend/src/lib/components/DateInput.svelte index 1881abcfcc..dec411f6cc 100644 --- a/frontend/src/lib/components/DateInput.svelte +++ b/frontend/src/lib/components/DateInput.svelte @@ -8,6 +8,7 @@ export let minDate: string | undefined = undefined export let maxDate: string | undefined = undefined export let dateFormat: string | undefined = 'dd-MM-yyyy' + export let disabled: boolean = false let date: string | undefined = computeDate(value) @@ -71,6 +72,7 @@
-
{k}
-
+
{k}
+
{#if min && total} -
+
{#each items?.[k] ?? [] as b} {@const waitingLen = b?.created_at ? b.started_at diff --git a/frontend/src/lib/components/LightweightArgInput.svelte b/frontend/src/lib/components/LightweightArgInput.svelte index ed7ea831fe..bc15a41926 100644 --- a/frontend/src/lib/components/LightweightArgInput.svelte +++ b/frontend/src/lib/components/LightweightArgInput.svelte @@ -36,6 +36,9 @@ export let type: string | undefined = undefined export let oneOf: SchemaProperty[] | undefined = undefined export let required = false + //todo + // export let nullable = false + export let disabled: boolean = false export let pattern: undefined | string = undefined export let valid = required ? false : true export let enum_: EnumType = undefined @@ -258,7 +261,7 @@
{#if inputCat == 'number'} {#if extra['min'] != undefined && extra['max'] != undefined} - + {:else if extra?.currency} {:else} {/if} {:else if inputCat == 'boolean'} @@ -295,6 +300,7 @@ ? '' : 'border !border-red-700 !border-opacity-70 focus:!border-red-700 focus:!border-opacity-30'} bind:checked={value} + {disabled} /> {#if type == 'boolean' && value == undefined}   Not set @@ -308,6 +314,7 @@ bind:selected={value} options={itemsType?.multiselect ?? []} selectedOptionsDraggable={true} + {disabled} />
{:else if Array.isArray(itemsType?.enum) && Array.isArray(value)} @@ -317,6 +324,7 @@ bind:selected={value} options={itemsType?.enum ?? []} selectedOptionsDraggable={true} + {disabled} />
{:else if Array.isArray(enum_) && Array.isArray(value)} @@ -326,6 +334,7 @@ bind:selected={value} options={enum_ ?? []} selectedOptionsDraggable={true} + {disabled} />
{:else} @@ -334,13 +343,14 @@ {#each value ?? [] as v, i}
{#if itemsType?.type == 'number'} - + {:else if itemsType?.type == 'string' && itemsType?.contentEncoding == 'base64'} fileChanged(x, (val) => (value[i] = val))} multiple={false} + {disabled} /> {:else if Array.isArray(itemsType?.enum)} + {/if} @@ -398,6 +412,7 @@ addItemByItemsType() }} startIcon={{ icon: Plus }} + {disabled} > Add @@ -430,13 +445,13 @@ />
{:else if inputCat == 'resource-object'} - + {:else if inputCat == 'object' && oneOf && oneOf.length >= 2}
{#if oneOf && oneOf.length >= 2} - + {#each oneOf as obj} - + {/each} @@ -455,6 +470,7 @@ type: 'object' }} bind:args={value} + {disabled} />
{/if} @@ -472,6 +488,7 @@ type: 'object' }} bind:args={value} + {disabled} />
{:else} @@ -487,6 +504,7 @@ : 'border !border-red-700 !border-opacity-70 focus:!border-red-700 focus:!border-opacity-30'}" placeholder={defaultValue ? JSON.stringify(defaultValue, null, 4) : ''} bind:value={rawValue} + {disabled} /> {/if} {:else if inputCat == 'enum'} @@ -496,6 +514,7 @@ }} class="px-6" bind:value + {disabled} > {#each enum_ ?? [] as e} @@ -504,9 +523,9 @@ {:else if inputCat == 'date'} {#key defaultChange} {#if format === 'date'} - + {:else} - + {/if} {/key} {:else if inputCat == 'base64'} @@ -515,6 +534,7 @@ type="file" on:change={(x) => fileChanged(x, (val) => (value = val))} multiple={false} + {disabled} /> {#if value?.length}
1 ? format.substring('resource-'.length) : undefined} + {disabled} />
{:else if inputCat == 'email'} @@ -542,6 +563,7 @@ : 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-3'} placeholder={placeholder ?? defaultValue ?? ''} bind:value + {disabled} /> {:else if inputCat == 'currency'} {:else if inputCat == 'string'}
{#if extra?.['password'] == true} - + {:else}