From 71d6bbbdc3f2350a12079ea5fe32da1183ce3cf2 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Thu, 17 Jul 2025 16:01:28 +0200 Subject: [PATCH] feat(cli): make `flow generate-locks` respect raw requirements (#6105) * build: add claude-code to nix shell Signed-off-by: pyranota * stage progress * stage files * make it work * stage work Signed-off-by: pyranota * make it work + refactor logic a bit Signed-off-by: pyranota * small cleanup Signed-off-by: pyranota * rename `local_lockfiles` to `rawReqs` Signed-off-by: pyranota * cleanup Signed-off-by: pyranota * fix transpilation to deno error Signed-off-by: pyranota * update sqlx Signed-off-by: pyranota * cleanup features.default Signed-off-by: pyranota * follow up rename Signed-off-by: pyranota * nits Signed-off-by: pyranota * overwrite flow.yaml on `flow generate-locks` Signed-off-by: pyranota * nits Signed-off-by: pyranota * remove invalid code generated by claude Signed-off-by: pyranota * nits Signed-off-by: pyranota * fix typo Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> * remove double quotes Signed-off-by: pyranota * use async to write file Signed-off-by: pyranota --------- Signed-off-by: pyranota Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> --- ...681862711af1e6b046c4ff479622c153b69ed.json | 68 ++++ ...31fb27912cfd05ba768a42cf9b3dfd7205e9a.json | 2 +- ...051f0760e22eccdd291c83ed86599ce571cb.json} | 4 +- ...4c398fe6398745e16fd119008b8ea67662416.json | 2 +- ...5199960264b54c1ddf928c0c654ab15176f63.json | 2 +- ...90203332e398c865c47aaf019dd5721d59f9.json} | 4 +- ...687ee8690d475b8ed31c48e31180689a278f.json} | 4 +- ...41d95ba195f5543120e76e68f6e6715b65a71.json | 2 +- ...7f5033b9c9afc344d9c3e385ba20a3ad2197a.json | 2 +- ...cabb01481a19bef5a956dc36cac41acad0e53.json | 2 +- ...4fbe9c20f9685b06ad5bbded424948b2dc88c.json | 2 +- ...17d660529c15b41860a4513f63d465bb9abb.json} | 4 +- ...80888d71121082390f3fdb91269cd0c5d0b2.json} | 4 +- ...329f748006ab131489d2fcac21d32641bb633.json | 2 +- ...20ae18f79cc84626097b766307cd42722baa3.json | 2 +- ...79115b6e9042800d40899a36b3867049c5c46.json | 2 +- ...b7a63521c1ad788033c564ffd7a2944454378.json | 68 ---- ...3b38deb9c7fa17bd3836c9ad8884f27be84a.json} | 4 +- ...b00147fc8bcd42f3df37ef81b9ded80974bd0.json | 2 +- ...cbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json} | 4 +- ...42b9fd3edd05c314c9d96ba33cad37fe35c2.json} | 4 +- ...de007185263167bf1dfa2e469d319cc4da674.json | 2 +- ...a0c5976ee60125289e9f7b0b04090bce4a3a3.json | 2 +- ...aa90d8d8c5c76f3971484e4c515ff60293d3b.json | 2 +- ...fe101cd5c2fe3e4c143c0b979668143d85dd2.json | 2 +- ...b322a3512ccfa86485e0f01664942ae04d0f.json} | 4 +- ...e9cc4e9aca7f9129b0466aacae51334a3db96.json | 2 +- backend/substitute_ee_code.sh | 2 +- backend/update_sqlx.sh | 4 +- backend/windmill-api/src/jobs.rs | 50 +-- backend/windmill-queue/src/jobs.rs | 5 +- backend/windmill-worker/src/worker.rs | 57 ++-- .../windmill-worker/src/worker_lockfiles.rs | 27 +- cli/build.sh | 2 +- cli/flow.ts | 15 +- cli/gen_wm_client.sh | 2 +- cli/metadata.ts | 314 +++++++++++------- cli/script.ts | 44 ++- cli/script_common.ts | 17 + cli/sync.ts | 3 +- cli/utils.ts | 2 +- flake.nix | 1 + 42 files changed, 433 insertions(+), 316 deletions(-) create mode 100644 backend/.sqlx/query-00ec0814dc270ee08d8cd818815681862711af1e6b046c4ff479622c153b69ed.json rename backend/.sqlx/{query-2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861.json => query-140f4ad799fca6c25975a0aca7c9051f0760e22eccdd291c83ed86599ce571cb.json} (67%) rename backend/.sqlx/{query-47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33.json => query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json} (76%) rename backend/.sqlx/{query-95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a.json => query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json} (72%) rename backend/.sqlx/{query-05cb171b610bfb45f6228128a385cde8a5b86d7ca377a028004cc382e12faf41.json => query-7b5dc50b685afc7de0b99bd68e2217d660529c15b41860a4513f63d465bb9abb.json} (63%) rename backend/.sqlx/{query-625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31.json => query-81ba1e003d3f6e8e7c74a4dd09b980888d71121082390f3fdb91269cd0c5d0b2.json} (52%) delete mode 100644 backend/.sqlx/query-b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378.json rename backend/.sqlx/{query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json => query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json} (60%) rename backend/.sqlx/{query-2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c.json => query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json} (72%) rename backend/.sqlx/{query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json => query-cad2dfc50d57c2298cb0ec05ed3b42b9fd3edd05c314c9d96ba33cad37fe35c2.json} (64%) rename backend/.sqlx/{query-70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836.json => query-f6e63a324664423353b56bccd9f3b322a3512ccfa86485e0f01664942ae04d0f.json} (78%) diff --git a/backend/.sqlx/query-00ec0814dc270ee08d8cd818815681862711af1e6b046c4ff479622c153b69ed.json b/backend/.sqlx/query-00ec0814dc270ee08d8cd818815681862711af1e6b046c4ff479622c153b69ed.json new file mode 100644 index 0000000000..6b5950359e --- /dev/null +++ b/backend/.sqlx/query-00ec0814dc270ee08d8cd818815681862711af1e6b046c4ff479622c153b69ed.json @@ -0,0 +1,68 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE\n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n CASE\n -- flow step:\n WHEN flow_step_id IS NOT NULL THEN NULL\n -- completed:\n WHEN c.id IS NOT NULL THEN COALESCE(\n c.workflow_as_code_status || c.flow_status,\n c.workflow_as_code_status,\n c.flow_status\n )\n -- not completed:\n ELSE COALESCE(\n f.workflow_as_code_status || f.flow_status,\n f.workflow_as_code_status,\n f.flow_status\n )\n END AS \"flow_status: sqlx::types::Json>\",\n job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "completed", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "running", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "logs", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "mem_peak", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 5, + "name": "log_offset", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "progress", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Text", + "Uuid", + "Bool", + "Bool" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + false, + null + ] + }, + "hash": "00ec0814dc270ee08d8cd818815681862711af1e6b046c4ff479622c153b69ed" +} diff --git a/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json b/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json index 1842f8ab06..66a2152ae9 100644 --- a/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json +++ b/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861.json b/backend/.sqlx/query-140f4ad799fca6c25975a0aca7c9051f0760e22eccdd291c83ed86599ce571cb.json similarity index 67% rename from backend/.sqlx/query-2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861.json rename to backend/.sqlx/query-140f4ad799fca6c25975a0aca7c9051f0760e22eccdd291c83ed86599ce571cb.json index 2ebfb49b8e..754eb02825 100644 --- a/backend/.sqlx/query-2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861.json +++ b/backend/.sqlx/query-140f4ad799fca6c25975a0aca7c9051f0760e22eccdd291c83ed86599ce571cb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_status (id, workflow_as_code_status)\n VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3))\n ON CONFLICT (id) DO UPDATE SET\n workflow_as_code_status = JSONB_SET(\n COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB), \n array[$2],\n $3\n )", + "query": "INSERT INTO v2_job_status (id, workflow_as_code_status)\n VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3))\n ON CONFLICT (id) DO UPDATE SET\n workflow_as_code_status = JSONB_SET(\n COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB),\n array[$2],\n $3\n )", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861" + "hash": "140f4ad799fca6c25975a0aca7c9051f0760e22eccdd291c83ed86599ce571cb" } diff --git a/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json b/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json index 8217ec4576..197298f788 100644 --- a/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json +++ b/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json b/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json index c4c33fe959..d64d8f404b 100644 --- a/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json +++ b/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33.json b/backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json similarity index 76% rename from backend/.sqlx/query-47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33.json rename to backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json index 9e1d80c2f6..08bb98cc06 100644 --- a/backend/.sqlx/query-47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33.json +++ b/backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\"\n FROM v2_as_completed_job \n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "query": "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\"\n FROM v2_as_completed_job\n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", "describe": { "columns": [ { @@ -26,5 +26,5 @@ true ] }, - "hash": "47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33" + "hash": "2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9" } diff --git a/backend/.sqlx/query-95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a.json b/backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json similarity index 72% rename from backend/.sqlx/query-95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a.json rename to backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json index d451727f46..ee5e15118d 100644 --- a/backend/.sqlx/query-95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a.json +++ b/backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_as_queue \n LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id \n WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", + "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_as_queue\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id\n WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", "describe": { "columns": [ { @@ -38,5 +38,5 @@ true ] }, - "hash": "95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a" + "hash": "2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f" } diff --git a/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json b/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json index e3b0e9a8da..31700319e2 100644 --- a/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json +++ b/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json index 1fa370e682..75b8108281 100644 --- a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json +++ b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json b/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json index 882342e23f..97079df64f 100644 --- a/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json +++ b/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json b/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json index 784a21b801..0f72e41b9f 100644 --- a/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json +++ b/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-05cb171b610bfb45f6228128a385cde8a5b86d7ca377a028004cc382e12faf41.json b/backend/.sqlx/query-7b5dc50b685afc7de0b99bd68e2217d660529c15b41860a4513f63d465bb9abb.json similarity index 63% rename from backend/.sqlx/query-05cb171b610bfb45f6228128a385cde8a5b86d7ca377a028004cc382e12faf41.json rename to backend/.sqlx/query-7b5dc50b685afc7de0b99bd68e2217d660529c15b41860a4513f63d465bb9abb.json index 5290fab4c4..7bd7c1a192 100644 --- a/backend/.sqlx/query-05cb171b610bfb45f6228128a385cde8a5b86d7ca377a028004cc382e12faf41.json +++ b/backend/.sqlx/query-7b5dc50b685afc7de0b99bd68e2217d660529c15b41860a4513f63d465bb9abb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO concurrency_counter(concurrency_id, job_uuids) \n VALUES ($1, '{}'::jsonb)", + "query": "INSERT INTO concurrency_counter(concurrency_id, job_uuids)\n VALUES ($1, '{}'::jsonb)", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "05cb171b610bfb45f6228128a385cde8a5b86d7ca377a028004cc382e12faf41" + "hash": "7b5dc50b685afc7de0b99bd68e2217d660529c15b41860a4513f63d465bb9abb" } diff --git a/backend/.sqlx/query-625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31.json b/backend/.sqlx/query-81ba1e003d3f6e8e7c74a4dd09b980888d71121082390f3fdb91269cd0c5d0b2.json similarity index 52% rename from backend/.sqlx/query-625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31.json rename to backend/.sqlx/query-81ba1e003d3f6e8e7c74a4dd09b980888d71121082390f3fdb91269cd0c5d0b2.json index 248f44b23a..83b4cc8e28 100644 --- a/backend/.sqlx/query-625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31.json +++ b/backend/.sqlx/query-81ba1e003d3f6e8e7c74a4dd09b980888d71121082390f3fdb91269cd0c5d0b2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n LEFT JOIN flow_version_lite \n ON flow_version_lite.id = flow_version.id\n WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1", + "query": "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow\n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n LEFT JOIN flow_version_lite\n ON flow_version_lite.id = flow_version.id\n WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1", "describe": { "columns": [ { @@ -19,5 +19,5 @@ null ] }, - "hash": "625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31" + "hash": "81ba1e003d3f6e8e7c74a4dd09b980888d71121082390f3fdb91269cd0c5d0b2" } diff --git a/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json b/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json index a60730197f..14d37809d5 100644 --- a/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json +++ b/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json b/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json index ec393b4218..acc6769652 100644 --- a/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json +++ b/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json b/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json index f8c8963a18..f38b293acb 100644 --- a/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json +++ b/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378.json b/backend/.sqlx/query-b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378.json deleted file mode 100644 index b495061906..0000000000 --- a/backend/.sqlx/query-b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE \n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n CASE\n -- flow step:\n WHEN flow_step_id IS NOT NULL THEN NULL\n -- completed:\n WHEN c.id IS NOT NULL THEN COALESCE(\n c.workflow_as_code_status || c.flow_status,\n c.workflow_as_code_status,\n c.flow_status\n )\n -- not completed:\n ELSE COALESCE(\n f.workflow_as_code_status || f.flow_status,\n f.workflow_as_code_status,\n f.flow_status\n )\n END AS \"flow_status: sqlx::types::Json>\",\n job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "completed", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "running", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "logs", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "mem_peak", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "flow_status: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 5, - "name": "log_offset", - "type_info": "Int4" - }, - { - "ordinal": 6, - "name": "created_by!", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "progress", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Text", - "Uuid", - "Bool", - "Bool" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null, - false, - null - ] - }, - "hash": "b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378" -} diff --git a/backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json b/backend/.sqlx/query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json similarity index 60% rename from backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json rename to backend/.sqlx/query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json index 8d51b798e9..4417c453f9 100644 --- a/backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json +++ b/backend/.sqlx/query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT \n j.id,\n j.kind AS \"kind: _\",\n COALESCE(s.path, f.path) AS \"script_path!\",\n COALESCE(s.hash, f.id) AS \"script_hash!: _\",\n COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS \"scheduled_for!: _\",\n args AS input,\n COALESCE(s.schema, f.schema) AS \"schema: _\"\n FROM v2_job j\n LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n LEFT JOIN v2_job_completed jc ON jc.id = j.id\n LEFT JOIN v2_job_queue jq ON jq.id = j.id\n WHERE j.id = ANY($1)\n AND j.workspace_id = $2\n AND COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL", + "query": "SELECT\n j.id,\n j.kind AS \"kind: _\",\n COALESCE(s.path, f.path) AS \"script_path!\",\n COALESCE(s.hash, f.id) AS \"script_hash!: _\",\n COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS \"scheduled_for!: _\",\n args AS input,\n COALESCE(s.schema, f.schema) AS \"schema: _\"\n FROM v2_job j\n LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n LEFT JOIN v2_job_completed jc ON jc.id = j.id\n LEFT JOIN v2_job_queue jq ON jq.id = j.id\n WHERE j.id = ANY($1)\n AND j.workspace_id = $2\n AND COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL", "describe": { "columns": [ { @@ -81,5 +81,5 @@ null ] }, - "hash": "ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9" + "hash": "b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a" } diff --git a/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json b/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json index b378d68f5b..4843d959c1 100644 --- a/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json +++ b/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c.json b/backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json similarity index 72% rename from backend/.sqlx/query-2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c.json rename to backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json index 6d916281eb..a76515fd7e 100644 --- a/backend/.sqlx/query-2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c.json +++ b/backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job \n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id \n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", + "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", "describe": { "columns": [ { @@ -38,5 +38,5 @@ true ] }, - "hash": "2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c" + "hash": "ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9" } diff --git a/backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json b/backend/.sqlx/query-cad2dfc50d57c2298cb0ec05ed3b42b9fd3edd05c314c9d96ba33cad37fe35c2.json similarity index 64% rename from backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json rename to backend/.sqlx/query-cad2dfc50d57c2298cb0ec05ed3b42b9fd3edd05c314c9d96ba33cad37fe35c2.json index 1ca60067d0..73bab032f1 100644 --- a/backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json +++ b/backend/.sqlx/query-cad2dfc50d57c2298cb0ec05ed3b42b9fd3edd05c314c9d96ba33cad37fe35c2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", + "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id)\n SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", "describe": { "columns": [], "parameters": { @@ -17,5 +17,5 @@ }, "nullable": [] }, - "hash": "fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa" + "hash": "cad2dfc50d57c2298cb0ec05ed3b42b9fd3edd05c314c9d96ba33cad37fe35c2" } diff --git a/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json b/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json index f550276925..912b9a53de 100644 --- a/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json +++ b/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json b/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json index 23b70ac4d2..79fe0873dd 100644 --- a/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json +++ b/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json b/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json index 1078957cf6..cdb8b928b3 100644 --- a/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json +++ b/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json b/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json index 90413de23c..a1ce88bd88 100644 --- a/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json +++ b/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836.json b/backend/.sqlx/query-f6e63a324664423353b56bccd9f3b322a3512ccfa86485e0f01664942ae04d0f.json similarity index 78% rename from backend/.sqlx/query-70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836.json rename to backend/.sqlx/query-f6e63a324664423353b56bccd9f3b322a3512ccfa86485e0f01664942ae04d0f.json index 9278984d51..4e21dac9a9 100644 --- a/backend/.sqlx/query-70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836.json +++ b/backend/.sqlx/query-f6e63a324664423353b56bccd9f3b322a3512ccfa86485e0f01664942ae04d0f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH uuid_table as (\n select unnest($4::uuid[]) as uuid\n )\n INSERT INTO v2_job_queue\n (id, workspace_id, scheduled_for, tag)\n (SELECT uuid, $1, $2, $3 FROM uuid_table) \n RETURNING id", + "query": "WITH uuid_table as (\n select unnest($4::uuid[]) as uuid\n )\n INSERT INTO v2_job_queue\n (id, workspace_id, scheduled_for, tag)\n (SELECT uuid, $1, $2, $3 FROM uuid_table)\n RETURNING id", "describe": { "columns": [ { @@ -21,5 +21,5 @@ false ] }, - "hash": "70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836" + "hash": "f6e63a324664423353b56bccd9f3b322a3512ccfa86485e0f01664942ae04d0f" } diff --git a/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json b/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json index 38f51ee611..091d205b40 100644 --- a/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json +++ b/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/substitute_ee_code.sh b/backend/substitute_ee_code.sh index a1819c75b2..3ea73e16cc 100755 --- a/backend/substitute_ee_code.sh +++ b/backend/substitute_ee_code.sh @@ -99,4 +99,4 @@ if [ "$MOVE_NEW_FILES" == "YES" ]; then echo "File moved '${ce_file}' -->> '${ee_file}'" fi done -fi \ No newline at end of file +fi diff --git a/backend/update_sqlx.sh b/backend/update_sqlx.sh index 6e64115ce0..d8bc473f29 100755 --- a/backend/update_sqlx.sh +++ b/backend/update_sqlx.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Default directory EE_DIR="../windmill-ee-private" @@ -32,4 +32,4 @@ if [[ "$(uname)" == "Darwin" ]]; then sed -i '' 's/^#samael = { version="0.0.14", features = \["xmlsec"\] }/samael = { version="0.0.14", features = ["xmlsec"] }/' Cargo.toml # Comment out the git-based samael dependency sed -i '' 's/^\(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/# \1/' Cargo.toml -fi \ No newline at end of file +fi diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index b178b20934..dd6309e99e 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -716,7 +716,7 @@ macro_rules! get_job_query { "SELECT \ {table}.id, {table}.workspace_id, parent_job, v2_job.created_by, v2_job.created_at, started_at, v2_job.runnable_id as script_hash, v2_job.runnable_path as script_path, \ CASE WHEN args is null THEN NULL - WHEN pg_column_size(args) < 90000 THEN + WHEN pg_column_size(args) < 90000 THEN CASE WHEN jsonb_typeof(args) = 'object' THEN args ELSE jsonb_build_object('value', args) END @@ -726,7 +726,7 @@ macro_rules! get_job_query { {flow} as raw_flow, flow_step_id IS NOT NULL AS is_flow_step, script_lang as language, \ {lock} as raw_lock, permissioned_as_email as email, visible_to_owner, memory_peak as mem_peak, v2_job.tag, v2_job.priority, preprocessed, worker,\ {additional_fields} \ - FROM {table} + FROM {table} INNER JOIN v2_job ON v2_job.id = {table}.id \ {additional_joins} \ LEFT JOIN job_logs ON {table}.id = job_id \ @@ -1135,8 +1135,8 @@ async fn get_job_logs( let record = sqlx::query!( "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index - FROM v2_as_completed_job - LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id + FROM v2_as_completed_job + LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", id, w_id, @@ -1179,8 +1179,8 @@ async fn get_job_logs( } else { let text = sqlx::query!( "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index - FROM v2_as_queue - LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id + FROM v2_as_queue + LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", id, w_id, @@ -1238,7 +1238,7 @@ async fn get_args( .flatten(); let record = sqlx::query!( "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\" - FROM v2_as_completed_job + FROM v2_as_completed_job WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", id, &w_id, @@ -3306,7 +3306,7 @@ fn batch_rerun_jobs_inner( tokio::spawn(async move { let mut job_stream = sqlx::query_as!( BatchReRunQueryReturnType, - r#"SELECT + r#"SELECT j.id, j.kind AS "kind: _", COALESCE(s.path, f.path) AS "script_path!", @@ -3921,7 +3921,7 @@ pub async fn run_workflow_as_code( VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3)) ON CONFLICT (id) DO UPDATE SET workflow_as_code_status = JSONB_SET( - COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB), + COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB), array[$2], $3 )", @@ -5111,11 +5111,7 @@ async fn run_dependencies_job( &db, PushIsolationLevel::IsolatedRoot(db.clone()), &w_id, - JobPayload::RawScriptDependencies { - script_path: script_path, - content: raw_code, - language: language, - }, + JobPayload::RawScriptDependencies { script_path, content: raw_code, language }, args, authed.display_username(), &authed.email, @@ -5147,6 +5143,7 @@ async fn run_dependencies_job( pub struct RunFlowDependenciesRequest { pub path: String, pub flow_value: FlowValue, + pub raw_deps: Option>, } #[derive(Serialize)] @@ -5166,15 +5163,20 @@ async fn run_flow_dependencies_job( )); } + // Create args HashMap with skip_flow_update and raw_deps if present + let mut args_map = HashMap::from([("skip_flow_update".to_string(), to_raw_value(&true))]); + + // Add raw_deps to args if present + if let Some(ref raw_deps) = req.raw_deps { + args_map.insert("raw_deps".to_string(), to_raw_value(raw_deps)); + } + let (uuid, tx) = push( &db, PushIsolationLevel::IsolatedRoot(db.clone()), &w_id, JobPayload::RawFlowDependencies { path: req.path, flow_value: req.flow_value }, - PushArgs::from(&HashMap::from([( - "skip_flow_update".to_string(), - to_raw_value(&true), - )])), + PushArgs::from(&args_map), authed.display_username(), &authed.email, username_to_permissioned_as(&authed.username), @@ -5305,10 +5307,10 @@ async fn add_batch_jobs( } else if let Some(path) = batch_info.path { let mut tx = user_db.clone().begin(&authed).await?; let value_json = sqlx::query!( - "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow + "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - LEFT JOIN flow_version_lite + LEFT JOIN flow_version_lite ON flow_version_lite.id = flow_version.id WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1", &path, &w_id @@ -5422,7 +5424,7 @@ async fn add_batch_jobs( ) INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag) - (SELECT uuid, $1, $2, $3 FROM uuid_table) + (SELECT uuid, $1, $2, $3 FROM uuid_table) RETURNING id"#, w_id, Utc::now(), @@ -5440,7 +5442,7 @@ async fn add_batch_jobs( .await?; sqlx::query!( - "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", &uuids, authed.email, @@ -5467,7 +5469,7 @@ async fn add_batch_jobs( if let Some(custom_concurrency_key) = custom_concurrency_key { sqlx::query!( - "INSERT INTO concurrency_counter(concurrency_id, job_uuids) + "INSERT INTO concurrency_counter(concurrency_id, job_uuids) VALUES ($1, '{}'::jsonb)", &custom_concurrency_key ) @@ -5744,7 +5746,7 @@ async fn get_job_update( let record = sqlx::query!( "SELECT c.id IS NOT NULL AS completed, - CASE + CASE WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END) ELSE false END AS running, diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index f6927df49f..0b8fbef7f8 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -3691,7 +3691,8 @@ pub async fn push<'c, 'd>( None, None, ), - JobPayload::RawFlowDependencies { path, flow_value } => ( + JobPayload::RawFlowDependencies { path, flow_value } => { + ( None, Some(path), None, @@ -3705,7 +3706,7 @@ pub async fn push<'c, 'd>( None, None, None, - ), + )}, JobPayload::FlowDependencies { path, dedicated_worker, version } => { // Keep inserting `value` if not all workers are updated. // Starting at `v1.440`, the value is fetched on pull from the version id. diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7945f1eea1..ab81a628aa 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -493,7 +493,10 @@ impl JobCompletedSender { } else { unbounded_sender } - .send_async(SendResult { result: SendResultPayload::JobCompleted(jc), time: Instant::now() }) + .send_async(SendResult { + result: SendResultPayload::JobCompleted(jc), + time: Instant::now(), + }) .await .map_err(|_e| { anyhow::anyhow!("Failed to send job completed to background processor") @@ -520,9 +523,13 @@ impl JobCompletedSender { match self { Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, .. }) => { if wait_for_capacity { - sender.send_async(SendResult { result: send_result, time: Instant::now() }).await + sender + .send_async(SendResult { result: send_result, time: Instant::now() }) + .await } else { - unbounded_sender.send_async(SendResult { result: send_result, time: Instant::now() }).await + unbounded_sender + .send_async(SendResult { result: send_result, time: Instant::now() }) + .await } } Self::Http(_) => { @@ -855,13 +862,9 @@ pub fn start_interactive_worker_shell( if now.duration_since(last).as_secs() > TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION => { - Duration::from_secs( - WORKER_SHELL_NAP_TIME_DURATION, - ) - } - _ => { - Duration::from_millis(*SLEEP_QUEUE * 10) + Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION) } + _ => Duration::from_millis(*SLEEP_QUEUE * 10), }; tokio::select! { _ = tokio::time::sleep(nap_time) => { @@ -869,7 +872,7 @@ pub fn start_interactive_worker_shell( _ = killpill_rx.recv() => { break; } - } + } } Err(err) => { @@ -1329,11 +1332,15 @@ pub async fn run_worker( let mut killed_but_draining_same_worker_jobs = false; let mut killpill_rx2 = killpill_rx.resubscribe(); - + loop { let last_processing_duration_secs = last_processing_duration.load(Ordering::SeqCst); if last_processing_duration_secs > 5 { - let sleep_duration = if last_processing_duration_secs > 10 { 10 } else { 5 }; + let sleep_duration = if last_processing_duration_secs > 10 { + 10 + } else { + 5 + }; tracing::warn!(worker = %worker_name, hostname = %hostname, "last bg processor processing duration > {sleep_duration}s: {last_processing_duration_secs}s, throttling next job pull by {sleep_duration}s"); last_processing_duration.store(0, Ordering::SeqCst); tokio::time::sleep(Duration::from_secs(sleep_duration)).await; @@ -1510,16 +1517,20 @@ pub async fn run_worker( last_suspend_first = Instant::now(); } - let job = match timeout(Duration::from_secs(10), pull( - &db, - suspend_first, - &worker_name, - None, - #[cfg(feature = "benchmark")] - &mut bench, + let job = match timeout( + Duration::from_secs(10), + pull( + &db, + suspend_first, + &worker_name, + None, + #[cfg(feature = "benchmark")] + &mut bench, + ) + .warn_after_seconds(2), ) - .warn_after_seconds(2)) - .await { + .await + { Ok(job) => job, Err(e) => { tracing::error!(worker = %worker_name, hostname = %hostname, "pull timed out after 10s, sleeping for 30s: {e:?}"); @@ -1743,7 +1754,9 @@ pub async fn run_worker( match symlink_dir(&windows_parent, &windows_target).await { Ok(_) => { - tracing::info!("Successfully created directory symlink on Windows"); + tracing::info!( + "Successfully created directory symlink on Windows" + ); } Err(e) => { tracing::warn!("Failed to create symlink_dir on Windows (likely needs admin privileges or Developer Mode): {}", e); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index ebdb155465..ffd6909032 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -695,6 +695,16 @@ pub async fn handle_flow_dependency_job( }) .flatten(); + let raw_deps = job + .args + .as_ref() + .map(|x| { + x.get("raw_deps") + .map(|v| serde_json::from_str::>(v.get()).ok()) + .flatten() + }) + .flatten(); + // `JobKind::FlowDependencies` job store either: // - A saved flow version `id` in the `script_hash` column. // - Preview raw flow in the `queue` or `job` table. @@ -741,6 +751,7 @@ pub async fn handle_flow_dependency_job( &nodes_to_relock, occupancy_metrics, skip_flow_update, + raw_deps, ) .await?; if !errors.is_empty() { @@ -910,6 +921,7 @@ async fn lock_modules<'c>( locks_to_reload: &Option>, occupancy_metrics: &mut OccupancyMetrics, skip_flow_update: bool, + raw_deps: Option>, // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) ) ) -> Result<( Vec, @@ -963,6 +975,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, + raw_deps.clone(), )) .await?; e.value = FlowModuleValue::ForloopFlow { @@ -998,6 +1011,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, + raw_deps.clone(), )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1025,6 +1039,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, + raw_deps.clone(), )) .await?; e.value = FlowModuleValue::WhileloopFlow { @@ -1057,6 +1072,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, + raw_deps.clone(), )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1082,6 +1098,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, + raw_deps.clone(), )) .await?; errors.extend(ninner_errors); @@ -1157,6 +1174,14 @@ async fn lock_modules<'c>( create_dir_all(job_dir).map_err(|e| { Error::ExecutionErr(format!("Error creating job dir for flow step lock: {e}")) })?; + + // If we have local lockfiles (and they are enabled) we will replace script content with lockfile and tell hander that it is raw_deps job + let (content, raw_deps) = raw_deps + .as_ref() + .and_then(|llfs| llfs.get(language.as_str())) + .map(|lock| (lock.to_owned(), true)) + .unwrap_or((content, false)); + let new_lock = capture_dependency_job( &job.id, &language, @@ -1174,7 +1199,7 @@ async fn lock_modules<'c>( "{}/flow", &path.clone().unwrap_or_else(|| job_path.to_string()) ), - false, + raw_deps, None, occupancy_metrics, ) diff --git a/cli/build.sh b/cli/build.sh index e228750b51..ef91f3a804 100755 --- a/cli/build.sh +++ b/cli/build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Note for mac OS users: you need to install gnu-sed with `brew install gnu-sed` and use `gsed` instead of `sed`. ./gen_wm_client.sh deno run -A dnt.ts diff --git a/cli/flow.ts b/cli/flow.ts index 798d294db7..8b44011b06 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -7,7 +7,7 @@ import * as wmill from "./gen/services.gen.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import { resolve, track_job } from "./script.ts"; import { defaultFlowDefinition } from "./bootstrap/flow_bootstrap.ts"; -import { generateFlowLockInternal } from "./metadata.ts"; +import { blueColor, generateFlowLockInternal } from "./metadata.ts"; import { SyncOptions, mergeConfigWithConfigFile } from "./conf.ts"; import { FSFSElement, elementsToMap, ignoreF } from "./sync.ts"; import { readInlinePathSync } from "./utils.ts"; @@ -238,15 +238,18 @@ async function run( async function generateLocks( opts: GlobalOptions & { yes?: boolean; + useRawRequirements?: boolean; } & SyncOptions, folder: string | undefined ) { + const useRawReqs = opts.useRawRequirements || Deno.env.get("USE_RAW_REQUIREMENTS") === "true"; + const workspace = await resolveWorkspace(opts); await requireLogin(opts); opts = await mergeConfigWithConfigFile(opts); if (folder) { // read script metadata file - await generateFlowLockInternal(folder, false, workspace); + await generateFlowLockInternal(folder, false, workspace, opts, undefined, undefined, useRawReqs); } else { const ignore = await ignoreF(opts); const elems = Object.keys( @@ -267,7 +270,7 @@ async function generateLocks( let hasAny = false; for (const folder of elems) { - const candidate = await generateFlowLockInternal(folder, true, workspace); + const candidate = await generateFlowLockInternal(folder, true, workspace, opts, undefined, undefined, useRawReqs); if (candidate) { hasAny = true; log.info(colors.green(`+ ${candidate}`)); @@ -289,7 +292,7 @@ async function generateLocks( return; } for (const folder of elems) { - await generateFlowLockInternal(folder, false, workspace); + await generateFlowLockInternal(folder, false, workspace, opts,undefined, undefined, useRawReqs); } } } @@ -348,6 +351,10 @@ const command = new Command() ) .arguments("[flow:file]") .option("--yes", "Skip confirmation prompt") + .option( + "-r --use-raw-requirements", + "Use raw requirements (requirements.txt, go.mod, package.json, etc) instead of generating them on the server (can also be set with USE_RAW_REQUIREMENTS=true environment variable)" + ) .option( "-i --includes ", "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)" diff --git a/cli/gen_wm_client.sh b/cli/gen_wm_client.sh index 21527acee9..1b36f489d9 100755 --- a/cli/gen_wm_client.sh +++ b/cli/gen_wm_client.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eou pipefail script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/cli/metadata.ts b/cli/metadata.ts index d2d37aa22d..dcf3cdef3b 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -14,9 +14,9 @@ import { } from "./bootstrap/script_bootstrap.ts"; import { Workspace } from "./workspace.ts"; import { SchemaProperty } from "./bootstrap/common.ts"; -import { ScriptLanguage } from "./script_common.ts"; +import { languagesWithRawReqsSupport, LanguageWithRawReqsSupport, ScriptLanguage } from "./script_common.ts"; import { inferContentTypeFromFilePath } from "./script_common.ts"; -import { GlobalDeps, exts } from "./script.ts"; +import { GlobalDeps, exts, findGlobalDeps } from "./script.ts"; import { FSFSElement, extractInlineScriptsForFlows, @@ -37,43 +37,16 @@ export class LockfileGenerationError extends Error { } } -export async function generateAllMetadata() {} +export async function generateAllMetadata() { } function findClosestRawReqs( - lang: "bun" | "python3" | "php" | "go" | undefined, + lang: LanguageWithRawReqsSupport | undefined, remotePath: string, - globalDeps: GlobalDeps + globalDeps: GlobalDeps, ): string | undefined { let bestCandidate: { k: string; v: string } | undefined = undefined; - if (lang == "bun") { - Object.entries(globalDeps.pkgs).forEach(([k, v]) => { - if ( - remotePath.startsWith(k) && - k.length >= (bestCandidate?.k ?? "").length - ) { - bestCandidate = { k, v }; - } - }); - } else if (lang == "python3") { - Object.entries(globalDeps.reqs).forEach(([k, v]) => { - if ( - remotePath.startsWith(k) && - k.length >= (bestCandidate?.k ?? "").length - ) { - bestCandidate = { k, v }; - } - }); - } else if (lang == "php") { - Object.entries(globalDeps.composers).forEach(([k, v]) => { - if ( - remotePath.startsWith(k) && - k.length >= (bestCandidate?.k ?? "").length - ) { - bestCandidate = { k, v }; - } - }); - } else if (lang == "go") { - Object.entries(globalDeps.goMods).forEach(([k, v]) => { + if (lang) { + Object.entries(globalDeps.get(lang) ?? {}).forEach(([k, v]) => { if ( remotePath.startsWith(k) && k.length >= (bestCandidate?.k ?? "").length @@ -87,12 +60,25 @@ function findClosestRawReqs( } const TOP_HASH = "__flow_hash"; -async function generateFlowHash(folder: string) { +async function generateFlowHash( + rawReqs: Record | undefined, + folder: string, + defaultTs: "bun" | "deno" | undefined +) { const elems = await FSFSElement(path.join(Deno.cwd(), folder), [], true); const hashes: Record = {}; for await (const f of elems.getChildren()) { if (exts.some((e) => f.path.endsWith(e))) { - hashes[f.path] = await generateHash(await f.getContentText()); + let reqs: string | undefined; + if (rawReqs) { + // Get language name from path + const lang = inferContentTypeFromFilePath(f.path, defaultTs); + // Get lock for that language + [, reqs] = Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? []; + } + + // Embed lock into hash + hashes[f.path] = await generateHash(await f.getContentText() + (reqs ?? "")); } } return { ...hashes, [TOP_HASH]: await generateHash(JSON.stringify(hashes)) }; @@ -101,9 +87,13 @@ export async function generateFlowLockInternal( folder: string, dryRun: boolean, workspace: Workspace, + opts: GlobalOptions & { + defaultTs?: "bun" | "deno"; + }, justUpdateMetadataLock?: boolean, - noStaleMessage?: boolean -): Promise { + noStaleMessage?: boolean, + useRawReqs?: boolean, +): Promise { if (folder.endsWith(SEP)) { folder = folder.substring(0, folder.length - 1); } @@ -114,7 +104,24 @@ export async function generateFlowLockInternal( log.info(`Generating lock for flow ${folder} at ${remote_path}`); } - let hashes = await generateFlowHash(folder); + let rawReqs: Record | undefined = undefined; + if (useRawReqs) { + // Find all dependency files in the workspace + const globalDeps = await findGlobalDeps(); + + // Find closest dependency files for this flow + rawReqs = {}; + + // TODO: PERF: Only include raw reqs for the languages that are in the flow + languagesWithRawReqsSupport.map((lang) => { + const dep = findClosestRawReqs(lang, folder, globalDeps); + if (dep) { + // @ts-ignore + rawReqs[lang.language] = dep; + } + }); + } + let hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs); const conf = await readLockfile(); if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) { @@ -128,8 +135,17 @@ export async function generateFlowLockInternal( return remote_path; } + if (useRawReqs) { + log.warn("If using local lockfiles, following redeployments from Web App will inevitably override generated lockfiles by CLI. To maintain your script's lockfiles you will need to redeploy only from CLI. (Behavior is subject to change)") + log.info( + (await blueColor())( + `Found raw requirements (${languagesWithRawReqsSupport.map((l) => l.rrFilename).join("/")}) for ${folder}, using it`, + ), + ); + } + const flowValue = (await yamlParseFile( - folder! + SEP + "flow.yaml" + folder! + SEP + "flow.yaml", )) as FlowFile; if (!justUpdateMetadataLock) { @@ -148,26 +164,40 @@ export async function generateFlowLockInternal( replaceInlineScripts( flowValue.value.modules, folder + SEP!, - changedScripts + changedScripts, ); + //removeChangedLocks - flowValue.value = await updateFlow(workspace, flowValue.value, remote_path); + flowValue.value = await updateFlow( + workspace, + flowValue.value, + remote_path, + rawReqs, + ); const inlineScripts = extractInlineScriptsForFlows( flowValue.value.modules, - newPathAssigner("bun") + newPathAssigner(opts.defaultTs ?? "bun"), ); inlineScripts .filter((s) => s.path.endsWith(".lock")) .forEach((s) => { Deno.writeTextFileSync( Deno.cwd() + SEP + folder + SEP + s.path, - s.content + s.content, ); }); + + // Overwrite `flow.yaml` with the new lockfile references + await Deno.writeTextFile( + Deno.cwd() + SEP + folder + SEP + "flow.yaml", + yamlStringify( + flowValue as Record + ) + ); } - hashes = await generateFlowHash(folder); + hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs); for (const [path, hash] of Object.entries(hashes)) { await updateMetadataGlobalLock(folder, hash, path); @@ -176,7 +206,7 @@ export async function generateFlowLockInternal( } // on windows, when using powershell, blue is not readable -async function blueColor(): Promise<(x: string) => void> { +export async function blueColor(): Promise<(x: string) => void> { const isWin = await getIsWin(); return isWin ? colors.black : colors.blue; } @@ -193,7 +223,7 @@ export async function generateScriptMetadataInternal( noStaleMessage: boolean, globalDeps: GlobalDeps, codebases: SyncCodebase[], - justUpdateMetadataLock?: boolean + justUpdateMetadataLock?: boolean, ): Promise { const remotePath = scriptPath .substring(0, scriptPath.indexOf(".")) @@ -201,23 +231,26 @@ export async function generateScriptMetadataInternal( const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs); + const rrLang = languagesWithRawReqsSupport.find((l) => language == l.language); + const rawReqs = findClosestRawReqs( - language as "bun" | "python3" | "php" | "go" | undefined, + rrLang, scriptPath, - globalDeps + globalDeps, ); - if (rawReqs) { + + if (rawReqs && rrLang) { log.info( (await blueColor())( - `Found raw requirements (package.json/requirements.txt/composer.json/go.mod) for ${scriptPath}, using it` - ) + `Found raw requirements (${rrLang.rrFilename}) for ${scriptPath}, using it`, + ), ); } const metadataWithType = await parseMetadataFile( remotePath, undefined, globalDeps, - codebases + codebases, ); // read script content @@ -229,7 +262,7 @@ export async function generateScriptMetadataInternal( if (await checkifMetadataUptodate(remotePath, hash, undefined)) { if (!noStaleMessage) { log.info( - colors.green(`Script ${remotePath} metadata is up-to-date, skipping`) + colors.green(`Script ${remotePath} metadata is up-to-date, skipping`), ); } return; @@ -251,7 +284,7 @@ export async function generateScriptMetadataInternal( scriptContent, language, metadataParsedContent, - scriptPath + scriptPath, ); } @@ -265,7 +298,7 @@ export async function generateScriptMetadataInternal( language, remotePath, metadataParsedContent, - rawReqs + rawReqs, ); } else { metadataParsedContent.lock = ""; @@ -286,7 +319,7 @@ export async function generateScriptMetadataInternal( hash = await generateScriptHash( rawReqs, scriptContent, - metadataContentUsedForHash + metadataContentUsedForHash, ); await updateMetadataGlobalLock(remotePath, hash); if (!justUpdateMetadataLock) { @@ -299,14 +332,14 @@ export async function updateScriptSchema( scriptContent: string, language: ScriptLanguage, metadataContent: Record, - path: string + path: string, ): Promise { // infer schema from script content and update it inplace const result = await inferSchema( language, scriptContent, metadataContent.schema, - path + path, ); metadataContent.schema = result.schema; if (result.has_preprocessor) { @@ -327,15 +360,12 @@ async function updateScriptLock( language: ScriptLanguage, remotePath: string, metadataContent: Record, - rawDeps: string | undefined + rawDeps: string | undefined, ): Promise { if ( !( - language == "bun" || - language == "python3" || - language == "go" || + languagesWithRawReqsSupport.some((l) => l.language == language) || language == "deno" || - language == "php" || language == "rust" || language == "ansible" ) @@ -363,7 +393,7 @@ async function updateScriptLock( raw_deps: rawDeps, entrypoint: remotePath, }), - } + }, ); let responseText = "reading response failed"; @@ -374,11 +404,11 @@ async function updateScriptLock( if (lock === undefined) { if (response?.["error"]?.["message"]) { throw new LockfileGenerationError( - `Failed to generate lockfile: ${response?.["error"]?.["message"]}` + `Failed to generate lockfile: ${response?.["error"]?.["message"]}`, ); } throw new LockfileGenerationError( - `Failed to generate lockfile: ${JSON.stringify(response, null, 2)}` + `Failed to generate lockfile: ${JSON.stringify(response, null, 2)}`, ); } const lockPath = remotePath + ".script.lock"; @@ -390,7 +420,7 @@ async function updateScriptLock( if (await Deno.stat(lockPath)) { await Deno.remove(lockPath); } - } catch {} + } catch { } metadataContent.lock = ""; } } catch (e) { @@ -398,7 +428,7 @@ async function updateScriptLock( throw e; } throw new LockfileGenerationError( - `Failed to generate lockfile:${rawResponse.statusText}, ${responseText}, ${e}` + `Failed to generate lockfile:${rawResponse.statusText}, ${responseText}, ${e}`, ); } } @@ -406,24 +436,48 @@ async function updateScriptLock( export async function updateFlow( workspace: Workspace, flow_value: FlowValue, - remotePath: string + remotePath: string, + rawDeps?: Record ): Promise { - // generate the script lock running a dependency job in Windmill and update it inplace - // TODO: update this once the client is released - const rawResponse = await fetch( - `${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`, - { - method: "POST", - headers: { - Cookie: `token=${workspace.token}`, - "Content-Type": "application/json", + let rawResponse; + + if (rawDeps != undefined) { + log.info(colors.blue("Using raw requirements for flow dependencies")); + + // generate the script lock running a dependency job in Windmill and update it inplace + rawResponse = await fetch( + `${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`, + { + method: "POST", + headers: { + Cookie: `token=${workspace.token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + flow_value, + path: remotePath, + use_local_lockfiles: true, + raw_deps: rawDeps, + }), }, - body: JSON.stringify({ - flow_value, - path: remotePath, - }), - } - ); + ); + } else { + // Standard dependency resolution on the server + rawResponse = await fetch( + `${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`, + { + method: "POST", + headers: { + Cookie: `token=${workspace.token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + flow_value, + path: remotePath, + }), + }, + ); + } let responseText = "reading response failed"; try { @@ -435,20 +489,20 @@ export async function updateFlow( const msg = (res as any)?.["error"]?.["message"]; if (msg) { throw new LockfileGenerationError( - `Failed to generate lockfile: ${msg}` + `Failed to generate lockfile: ${msg}`, ); } throw new LockfileGenerationError( - `Failed to generate lockfile: ${rawResponse.statusText}, ${responseText}` + `Failed to generate lockfile: ${rawResponse.statusText}, ${responseText}`, ); } return (res as any).updated_flow_value; } catch (e) { try { responseText = await rawResponse.text(); - } catch {} + } catch { } throw new Error( - `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}` + `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`, ); } } @@ -460,7 +514,7 @@ export async function inferSchema( language: ScriptLanguage, content: string, currentSchema: any, - path: string + path: string, ): Promise<{ schema: any; has_preprocessor: boolean | undefined; @@ -586,8 +640,8 @@ export async function inferSchema( if (inferedSchema.type == "Invalid") { log.info( colors.yellow( - `Script ${path} invalid, it cannot be parsed to infer schema.` - ) + `Script ${path} invalid, it cannot be parsed to infer schema.`, + ), ); return { schema: defaultScriptMetadata().schema, @@ -601,7 +655,7 @@ export async function inferSchema( } currentSchema.required = []; const oldProperties = JSON.parse( - JSON.stringify(currentSchema?.properties ?? {}) + JSON.stringify(currentSchema?.properties ?? {}), ); currentSchema.properties = {}; @@ -612,7 +666,7 @@ export async function inferSchema( currentSchema.properties[arg.name] = oldProperties[arg.name]; } currentSchema.properties[arg.name] = sortObject( - currentSchema.properties[arg.name] + currentSchema.properties[arg.name], ); argSigToJsonSchemaType(arg.typ, currentSchema.properties[arg.name]); @@ -639,7 +693,7 @@ function sortObject(obj: any): any { ...acc, [key]: obj[key], }), - {} + {}, ); } @@ -649,24 +703,24 @@ export function argSigToJsonSchemaType( | string | { resource: string | null } | { - list: - | (string | { object: { key: string; typ: any }[] }) - | { str: any } - | { object: { key: string; typ: any }[] } - | null; - } + list: + | (string | { object: { key: string; typ: any }[] }) + | { str: any } + | { object: { key: string; typ: any }[] } + | null; + } | { dynselect: string } | { str: string[] | null } | { object: { key: string; typ: any }[] } | { - oneof: [ - { - label: string; - properties: { key: string; typ: any }[]; - } - ]; - }, - oldS: SchemaProperty + oneof: [ + { + label: string; + properties: { key: string; typ: any }[]; + }, + ]; + }, + oldS: SchemaProperty, ): void { const newS: SchemaProperty = { type: "" }; if (t === "int") { @@ -858,7 +912,9 @@ export function replaceLock(o?: { lock?: string | string[] }) { o.lock = readInlinePathSync(lockPath); } catch (e) { log.info( - colors.yellow(`Failed to read lockfile, doing as if it was empty: ${e}`) + colors.yellow( + `Failed to read lockfile, doing as if it was empty: ${e}`, + ), ); o.lock = ""; } @@ -868,13 +924,13 @@ export async function parseMetadataFile( scriptPath: string, generateMetadataIfMissing: | (GlobalOptions & { - path: string; - workspaceRemote: Workspace; - schemaOnly?: boolean; - }) + path: string; + workspaceRemote: Workspace; + schemaOnly?: boolean; + }) | undefined, globalDeps: GlobalDeps, - codebases: SyncCodebase[] + codebases: SyncCodebase[], ): Promise<{ isJson: boolean; payload: any; path: string }> { let metadataFilePath = scriptPath + ".script.json"; try { @@ -900,14 +956,14 @@ export async function parseMetadataFile( // no metadata file at all. Create it log.info( (await blueColor())( - `Creating script metadata file for ${metadataFilePath}` - ) + `Creating script metadata file for ${metadataFilePath}`, + ), ); metadataFilePath = scriptPath + ".script.yaml"; let scriptInitialMetadata = defaultScriptMetadata(); const scriptInitialMetadataYaml = yamlStringify( scriptInitialMetadata as Record, - yamlOptions + yamlOptions, ); await Deno.writeTextFile(metadataFilePath, scriptInitialMetadataYaml, { createNew: true, @@ -916,8 +972,8 @@ export async function parseMetadataFile( if (generateMetadataIfMissing) { log.info( (await blueColor())( - `Generating lockfile and schema for ${metadataFilePath}` - ) + `Generating lockfile and schema for ${metadataFilePath}`, + ), ); try { await generateScriptMetadataInternal( @@ -928,10 +984,10 @@ export async function parseMetadataFile( false, globalDeps, codebases, - false + false, ); scriptInitialMetadata = (await yamlParseFile( - metadataFilePath + metadataFilePath, )) as ScriptMetadata; if (!generateMetadataIfMissing.schemaOnly) { replaceLock(scriptInitialMetadata); @@ -939,8 +995,8 @@ export async function parseMetadataFile( } catch (e) { log.info( colors.yellow( - `Failed to generate lockfile and schema for ${metadataFilePath}: ${e}` - ) + `Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`, + ), ); } } @@ -979,7 +1035,7 @@ export async function checkifMetadataUptodate( path: string, hash: string, conf: Lock | undefined, - subpath?: string + subpath?: string, ) { if (!conf) { conf = await readLockfile(); @@ -995,17 +1051,17 @@ export async function checkifMetadataUptodate( export async function generateScriptHash( rawReqs: string | undefined, scriptContent: string, - newMetadataContent: string + newMetadataContent: string, ) { return await generateHash( - (rawReqs ?? "") + scriptContent + newMetadataContent + (rawReqs ?? "") + scriptContent + newMetadataContent, ); } export async function updateMetadataGlobalLock( path: string, hash: string, - subpath?: string + subpath?: string, ): Promise { const conf = await readLockfile(); if (!conf?.locks) { @@ -1024,6 +1080,6 @@ export async function updateMetadataGlobalLock( } await Deno.writeTextFile( WMILL_LOCKFILE, - yamlStringify(conf as Record, yamlOptions) + yamlStringify(conf as Record, yamlOptions), ); } diff --git a/cli/script.ts b/cli/script.ts index 87e4f85354..b5d978e13f 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -26,8 +26,10 @@ import { parseMetadataFile, } from "./metadata.ts"; import { + LanguageWithRawReqsSupport, ScriptLanguage, inferContentTypeFromFilePath, + languagesWithRawReqsSupport, } from "./script_common.ts"; import { elementsToMap, @@ -865,43 +867,35 @@ async function bootstrap( ); } -export type GlobalDeps = { - pkgs: Record; - reqs: Record; - composers: Record; - goMods: Record; -}; +export type GlobalDeps = Map>; + export async function findGlobalDeps(): Promise { - const pkgs: { [key: string]: string } = {}; - const reqs: { [key: string]: string } = {}; - const composers: { [key: string]: string } = {}; - const goMods: { [key: string]: string } = {}; + var globalDeps: GlobalDeps = new Map(); const els = await FSFSElement(Deno.cwd(), [], false); for await (const entry of readDirRecursiveWithIgnore((p, isDir) => { p = SEP + p; return ( !isDir && - !( - p.endsWith(SEP + "package.json") || - p.endsWith(SEP + "requirements.txt") || - p.endsWith(SEP + "composer.json") || - p.endsWith(SEP + "go.mod") + // Skip if the filename is not one of lockfile names + !(languagesWithRawReqsSupport.some( + lockfile => + p.endsWith(SEP + lockfile.rrFilename)) ) ); }, els)) { if (entry.isDirectory || entry.ignored) continue; const content = await entry.getContentText(); - if (entry.path.endsWith("package.json")) { - pkgs[entry.path.substring(0, entry.path.length - "package.json".length)] = content; - } else if (entry.path.endsWith("requirements.txt")) { - reqs[entry.path.substring(0, entry.path.length - "requirements.txt".length)] = content; - } else if (entry.path.endsWith("composer.json")) { - composers[entry.path.substring(0, entry.path.length - "composer.json".length)] = content; - } else if (entry.path.endsWith("go.mod")) { - goMods[entry.path.substring(0, entry.path.length - "go.mod".length)] = content; - } + + // Iterate over available languages to find which lockfile + languagesWithRawReqsSupport.map((lock) => { + if (entry.path.endsWith(lock.rrFilename)){ + const current = globalDeps.get(lock) ?? {}; + current[entry.path.substring(0, entry.path.length - lock.rrFilename.length)] = content; + globalDeps.set(lock, current); + } + }); } - return { pkgs, reqs, composers, goMods }; + return globalDeps; } async function generateMetadata( opts: GlobalOptions & { diff --git a/cli/script_common.ts b/cli/script_common.ts index 39906e7bb9..85efea9786 100644 --- a/cli/script_common.ts +++ b/cli/script_common.ts @@ -22,6 +22,23 @@ export type ScriptLanguage = | "java"; // for related places search: ADD_NEW_LANG +// To make language support raw requirements: +// 1. Add value here +// 2. Modify backend to allow raw deps +export type LanguageWithRawReqsSupport = + | { language: "bun", rrFilename /** (raw requirements filename) */: "package.json" } + // TODO: Add `requirements.in` - more intuitive and reflects better what actually happens + | { language: "python3", rrFilename: "requirements.txt" } + | { language: "php", rrFilename: "composer.json" } + | { language: "go", rrFilename: "go.mod" }; + +export const languagesWithRawReqsSupport: LanguageWithRawReqsSupport[] = [ + { language: "bun", rrFilename: "package.json" }, + { language: "python3", rrFilename: "requirements.txt" }, + { language: "php", rrFilename: "composer.json" }, + { language: "go", rrFilename: "go.mod" }, +] as const; + export function inferContentTypeFromFilePath( contentPath: string, defaultTs: "bun" | "deno" | undefined diff --git a/cli/sync.ts b/cli/sync.ts index 8ac9306a11..2fe403e2c5 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -1491,7 +1491,7 @@ export async function pull(opts: GlobalOptions & SyncOptions & { repository?: st } for (const change of tracker.flows) { log.info(`Updating lock for flow ${change}`); - await generateFlowLockInternal(change, false, workspace, true); + await generateFlowLockInternal(change, false, workspace, opts, true); } if (tracker.apps.length > 0) { log.info( @@ -1686,6 +1686,7 @@ export async function push(opts: GlobalOptions & SyncOptions & { repository?: st change, true, workspace, + opts, false, true ); diff --git a/cli/utils.ts b/cli/utils.ts index 1defd68b3f..561cb6504b 100644 --- a/cli/utils.ts +++ b/cli/utils.ts @@ -196,4 +196,4 @@ export async function selectRepository( }); return repositories.find((r) => r.git_repo_resource_path === selectedRepo)!; -} \ No newline at end of file +} diff --git a/flake.nix b/flake.nix index 34bad7cef7..0a8f3b9138 100644 --- a/flake.nix +++ b/flake.nix @@ -211,6 +211,7 @@ DENO_PATH = "${pkgs.deno}/bin/deno"; GO_PATH = "${pkgs.go}/bin/go"; PHP_PATH = "${pkgs.php}/bin/php"; + COMPOSER_PATH = "${pkgs.php84Packages.composer}/bin/composer"; BUN_PATH = "${pkgs.bun}/bin/bun"; UV_PATH = "${pkgs.uv}/bin/uv"; NU_PATH = "${pkgs.nushell}/bin/nu";