From 02e38b4463560338817c6ec121d745365a95c2c4 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 1 Dec 2025 15:06:48 +0100 Subject: [PATCH 01/10] feat(cli): jobs migration command (#7268) --- ...6b319c4bac92cb2d90bb4ffd08e25f4a4d18b.json | 15 + ...82655d08946206dcfb25158c10f78619cf7fc.json | 14 + ...b2892b100a6ca1c99396254e24b981a2bb1c0.json | 15 + ...66cc04d41f6d8917c3c0873304250d39943ff.json | 15 + ...2271eaaddde1f4e3a4ab62628faee2e06e947.json | 26 + ...52ca3f1137dd26c85963863f4a73150fcc0e0.json | 15 + ...1038b29ca689dd87b800cfd95f463592e382c.json | 14 + ...84b2c6723c5568940d2fdba85ce4800d8c5ca.json | 42 + ...ba728eb654566d3f1d9d9711f4340fa1b47fd.json | 15 + ...53492943c961f98036133437ccc1a621ee89c.json | 18 + ...03041f8a893fe9ed2830aecaa4d245b07d739.json | 16 + ...6a92a21355b0c153db26ac01e93509aa5a6e4.json | 14 + ...e33244fe8ab9a803248fc23f334034d24aad4.json | 381 ++++++++ ...be955155b7d0344dfa908451faf2a0ba9269b.json | 127 +++ ...671dd89e10000d3313884a663852337a76b2d.json | 17 + ...b992d8fa53ec801067e5b6c45daf05e61d136.json | 15 + ...2239b1cd80d969fa36c9cf3dee0c360dec302.json | 14 + ...9352d4d1e3cb3a8a08f1c9c03055a1cab1235.json | 411 +++++++++ ...b691ef7d1b0ad9eb2596ccfa7227721a0784f.json | 127 +++ ...9080006f1c36a53e1e9e631d794a876d5e0b3.json | 15 + ...ca562984f8b86c2882becef42e838bf79375d.json | 14 + backend/windmill-api/openapi.yaml | 406 +++++++++ backend/windmill-api/src/jobs.rs | 17 + backend/windmill-api/src/jobs_export.rs | 828 ++++++++++++++++++ backend/windmill-api/src/lib.rs | 1 + backend/windmill-common/src/jobs.rs | 10 + cli/src/commands/jobs/jobs.ts | 286 ++++++ cli/src/main.ts | 2 + 28 files changed, 2890 insertions(+) create mode 100644 backend/.sqlx/query-023555d33652d40fa381b1baaae6b319c4bac92cb2d90bb4ffd08e25f4a4d18b.json create mode 100644 backend/.sqlx/query-02fdd7b94e6b6c9bb7985dfeb2082655d08946206dcfb25158c10f78619cf7fc.json create mode 100644 backend/.sqlx/query-13aaff548ff7d3bd38c40799958b2892b100a6ca1c99396254e24b981a2bb1c0.json create mode 100644 backend/.sqlx/query-3bd11d7210bb548df4f18f980ae66cc04d41f6d8917c3c0873304250d39943ff.json create mode 100644 backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json create mode 100644 backend/.sqlx/query-409b452cfe4d1e36b2152e86ada52ca3f1137dd26c85963863f4a73150fcc0e0.json create mode 100644 backend/.sqlx/query-504bc8de80cda9c5db7db4e34a01038b29ca689dd87b800cfd95f463592e382c.json create mode 100644 backend/.sqlx/query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json create mode 100644 backend/.sqlx/query-82e1ad04f4030fd20e4738b3cebba728eb654566d3f1d9d9711f4340fa1b47fd.json create mode 100644 backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json create mode 100644 backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json create mode 100644 backend/.sqlx/query-9d1585d6fccc3c76eb2db98789d6a92a21355b0c153db26ac01e93509aa5a6e4.json create mode 100644 backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json create mode 100644 backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json create mode 100644 backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json create mode 100644 backend/.sqlx/query-cca63828cbb043d2c647830cf50b992d8fa53ec801067e5b6c45daf05e61d136.json create mode 100644 backend/.sqlx/query-d3038367ca90595eb165bcb6d172239b1cd80d969fa36c9cf3dee0c360dec302.json create mode 100644 backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json create mode 100644 backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json create mode 100644 backend/.sqlx/query-fae4c8069ef5389e64a35d3867e9080006f1c36a53e1e9e631d794a876d5e0b3.json create mode 100644 backend/.sqlx/query-fc191f8598ce030fdee04353ad6ca562984f8b86c2882becef42e838bf79375d.json create mode 100644 backend/windmill-api/src/jobs_export.rs create mode 100644 cli/src/commands/jobs/jobs.ts diff --git a/backend/.sqlx/query-023555d33652d40fa381b1baaae6b319c4bac92cb2d90bb4ffd08e25f4a4d18b.json b/backend/.sqlx/query-023555d33652d40fa381b1baaae6b319c4bac92cb2d90bb4ffd08e25f4a4d18b.json new file mode 100644 index 0000000000..e44ee4d255 --- /dev/null +++ b/backend/.sqlx/query-023555d33652d40fa381b1baaae6b319c4bac92cb2d90bb4ffd08e25f4a4d18b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job WHERE workspace_id = $1 AND id = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "023555d33652d40fa381b1baaae6b319c4bac92cb2d90bb4ffd08e25f4a4d18b" +} diff --git a/backend/.sqlx/query-02fdd7b94e6b6c9bb7985dfeb2082655d08946206dcfb25158c10f78619cf7fc.json b/backend/.sqlx/query-02fdd7b94e6b6c9bb7985dfeb2082655d08946206dcfb25158c10f78619cf7fc.json new file mode 100644 index 0000000000..884b7ff354 --- /dev/null +++ b/backend/.sqlx/query-02fdd7b94e6b6c9bb7985dfeb2082655d08946206dcfb25158c10f78619cf7fc.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_status WHERE id = ANY($1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "02fdd7b94e6b6c9bb7985dfeb2082655d08946206dcfb25158c10f78619cf7fc" +} diff --git a/backend/.sqlx/query-13aaff548ff7d3bd38c40799958b2892b100a6ca1c99396254e24b981a2bb1c0.json b/backend/.sqlx/query-13aaff548ff7d3bd38c40799958b2892b100a6ca1c99396254e24b981a2bb1c0.json new file mode 100644 index 0000000000..ecd2d79a53 --- /dev/null +++ b/backend/.sqlx/query-13aaff548ff7d3bd38c40799958b2892b100a6ca1c99396254e24b981a2bb1c0.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM job_stats WHERE workspace_id = $1 AND job_id = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "13aaff548ff7d3bd38c40799958b2892b100a6ca1c99396254e24b981a2bb1c0" +} diff --git a/backend/.sqlx/query-3bd11d7210bb548df4f18f980ae66cc04d41f6d8917c3c0873304250d39943ff.json b/backend/.sqlx/query-3bd11d7210bb548df4f18f980ae66cc04d41f6d8917c3c0873304250d39943ff.json new file mode 100644 index 0000000000..a2c0c127cd --- /dev/null +++ b/backend/.sqlx/query-3bd11d7210bb548df4f18f980ae66cc04d41f6d8917c3c0873304250d39943ff.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM job_logs WHERE workspace_id = $1 AND job_id = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "3bd11d7210bb548df4f18f980ae66cc04d41f6d8917c3c0873304250d39943ff" +} diff --git a/backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json b/backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json new file mode 100644 index 0000000000..e74dc52790 --- /dev/null +++ b/backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job_queue (\n id,\n workspace_id,\n started_at,\n scheduled_for,\n running,\n canceled_by,\n canceled_reason,\n suspend,\n suspend_until,\n worker,\n extras,\n tag,\n priority\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13\n )\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Timestamptz", + "Timestamptz", + "Bool", + "Varchar", + "Text", + "Int4", + "Timestamptz", + "Varchar", + "Jsonb", + "Varchar", + "Int2" + ] + }, + "nullable": [] + }, + "hash": "3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947" +} diff --git a/backend/.sqlx/query-409b452cfe4d1e36b2152e86ada52ca3f1137dd26c85963863f4a73150fcc0e0.json b/backend/.sqlx/query-409b452cfe4d1e36b2152e86ada52ca3f1137dd26c85963863f4a73150fcc0e0.json new file mode 100644 index 0000000000..526adab016 --- /dev/null +++ b/backend/.sqlx/query-409b452cfe4d1e36b2152e86ada52ca3f1137dd26c85963863f4a73150fcc0e0.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_queue WHERE workspace_id = $1 AND id = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "409b452cfe4d1e36b2152e86ada52ca3f1137dd26c85963863f4a73150fcc0e0" +} diff --git a/backend/.sqlx/query-504bc8de80cda9c5db7db4e34a01038b29ca689dd87b800cfd95f463592e382c.json b/backend/.sqlx/query-504bc8de80cda9c5db7db4e34a01038b29ca689dd87b800cfd95f463592e382c.json new file mode 100644 index 0000000000..785d393abe --- /dev/null +++ b/backend/.sqlx/query-504bc8de80cda9c5db7db4e34a01038b29ca689dd87b800cfd95f463592e382c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM resume_job WHERE job = ANY($1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "504bc8de80cda9c5db7db4e34a01038b29ca689dd87b800cfd95f463592e382c" +} diff --git a/backend/.sqlx/query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json b/backend/.sqlx/query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json new file mode 100644 index 0000000000..9fe2580534 --- /dev/null +++ b/backend/.sqlx/query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json @@ -0,0 +1,42 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job_completed (\n id,\n workspace_id,\n started_at,\n completed_at,\n duration_ms,\n result,\n deleted,\n canceled_by,\n canceled_reason,\n flow_status,\n memory_peak,\n status,\n worker,\n workflow_as_code_status,\n result_columns,\n retries,\n extras\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17\n )\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Timestamptz", + "Timestamptz", + "Int8", + "Jsonb", + "Bool", + "Varchar", + "Text", + "Jsonb", + "Int4", + { + "Custom": { + "name": "job_status", + "kind": { + "Enum": [ + "success", + "failure", + "canceled", + "skipped" + ] + } + } + }, + "Varchar", + "Jsonb", + "TextArray", + "UuidArray", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca" +} diff --git a/backend/.sqlx/query-82e1ad04f4030fd20e4738b3cebba728eb654566d3f1d9d9711f4340fa1b47fd.json b/backend/.sqlx/query-82e1ad04f4030fd20e4738b3cebba728eb654566d3f1d9d9711f4340fa1b47fd.json new file mode 100644 index 0000000000..9b34ff6662 --- /dev/null +++ b/backend/.sqlx/query-82e1ad04f4030fd20e4738b3cebba728eb654566d3f1d9d9711f4340fa1b47fd.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH inserted_concurrency_counter AS (\n INSERT INTO concurrency_counter (concurrency_id, job_uuids)\n VALUES ($1, '{}'::jsonb)\n ON CONFLICT DO NOTHING\n )\n INSERT INTO concurrency_key(key, job_id)\n VALUES ($1, $2)\n ON CONFLICT (job_id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "82e1ad04f4030fd20e4738b3cebba728eb654566d3f1d9d9711f4340fa1b47fd" +} diff --git a/backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json b/backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json new file mode 100644 index 0000000000..5979389314 --- /dev/null +++ b/backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO job_logs (\n job_id,\n workspace_id,\n logs,\n log_offset,\n log_file_index\n ) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (job_id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Text", + "Int4", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c" +} diff --git a/backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json b/backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json new file mode 100644 index 0000000000..088d2317c9 --- /dev/null +++ b/backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job_runtime (\n id, ping, memory_peak\n ) VALUES (\n $1, $2, $3\n )\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Timestamptz", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739" +} diff --git a/backend/.sqlx/query-9d1585d6fccc3c76eb2db98789d6a92a21355b0c153db26ac01e93509aa5a6e4.json b/backend/.sqlx/query-9d1585d6fccc3c76eb2db98789d6a92a21355b0c153db26ac01e93509aa5a6e4.json new file mode 100644 index 0000000000..41dc5084ce --- /dev/null +++ b/backend/.sqlx/query-9d1585d6fccc3c76eb2db98789d6a92a21355b0c153db26ac01e93509aa5a6e4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM concurrency_key WHERE job_id = ANY($1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "9d1585d6fccc3c76eb2db98789d6a92a21355b0c153db26ac01e93509aa5a6e4" +} diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json new file mode 100644 index 0000000000..c3c57b081c --- /dev/null +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -0,0 +1,381 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n v2_job.id,\n v2_job.raw_code,\n v2_job.raw_lock,\n v2_job.raw_flow as \"raw_flow: _\",\n v2_job.tag,\n v2_job.created_at,\n v2_job.created_by,\n v2_job.permissioned_as,\n v2_job.permissioned_as_email,\n v2_job.kind as \"kind: _\",\n v2_job.runnable_id,\n v2_job.runnable_path,\n v2_job.parent_job,\n v2_job.root_job,\n v2_job.script_lang as \"script_lang: _\",\n v2_job.script_entrypoint_override,\n v2_job.flow_step,\n v2_job.flow_step_id,\n v2_job.flow_innermost_root_job,\n v2_job.\"trigger\",\n v2_job.trigger_kind as \"trigger_kind: _\",\n v2_job.same_worker,\n v2_job.visible_to_owner,\n v2_job.concurrent_limit,\n v2_job.concurrency_time_window_s,\n v2_job.cache_ttl,\n v2_job.timeout,\n v2_job.priority,\n v2_job.preprocessed,\n v2_job.args as \"args: _\",\n v2_job.labels,\n v2_job.pre_run_error,\n\n v2_job_queue.started_at,\n v2_job_queue.scheduled_for,\n v2_job_queue.running,\n v2_job_queue.canceled_by,\n v2_job_queue.canceled_reason,\n v2_job_queue.suspend,\n v2_job_queue.suspend_until,\n v2_job_queue.worker,\n v2_job_queue.extras as \"extras: _\",\n\n v2_job_runtime.ping,\n v2_job_runtime.memory_peak,\n\n v2_job_status.flow_status as \"flow_status: _\",\n v2_job_status.flow_leaf_jobs as \"flow_leaf_jobs: _\",\n v2_job_status.workflow_as_code_status as \"workflow_as_code_status: _\",\n\n concurrency_key.key as \"concurrency_key?\"\n FROM v2_job_queue\n INNER JOIN v2_job ON v2_job.id = v2_job_queue.id\n LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id\n LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id\n LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id\n WHERE v2_job_queue.workspace_id = $1\n AND v2_job_queue.running = false\n AND v2_job.parent_job IS NULL\n AND v2_job.trigger_kind IS DISTINCT FROM 'schedule'\n ORDER BY v2_job.created_at DESC\n LIMIT $2\n OFFSET $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "raw_code", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "raw_lock", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "raw_flow: _", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "kind: _", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned" + ] + } + } + } + }, + { + "ordinal": 10, + "name": "runnable_id", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 12, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 13, + "name": "root_job", + "type_info": "Uuid" + }, + { + "ordinal": 14, + "name": "script_lang: _", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "script_entrypoint_override", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "flow_step", + "type_info": "Int4" + }, + { + "ordinal": 17, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 18, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 19, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 20, + "name": "trigger_kind: _", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud" + ] + } + } + } + }, + { + "ordinal": 21, + "name": "same_worker", + "type_info": "Bool" + }, + { + "ordinal": 22, + "name": "visible_to_owner", + "type_info": "Bool" + }, + { + "ordinal": 23, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 24, + "name": "concurrency_time_window_s", + "type_info": "Int4" + }, + { + "ordinal": 25, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 26, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 27, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 28, + "name": "preprocessed", + "type_info": "Bool" + }, + { + "ordinal": 29, + "name": "args: _", + "type_info": "Jsonb" + }, + { + "ordinal": 30, + "name": "labels", + "type_info": "TextArray" + }, + { + "ordinal": 31, + "name": "pre_run_error", + "type_info": "Text" + }, + { + "ordinal": 32, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 33, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 34, + "name": "running", + "type_info": "Bool" + }, + { + "ordinal": 35, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 36, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 37, + "name": "suspend", + "type_info": "Int4" + }, + { + "ordinal": 38, + "name": "suspend_until", + "type_info": "Timestamptz" + }, + { + "ordinal": 39, + "name": "worker", + "type_info": "Varchar" + }, + { + "ordinal": 40, + "name": "extras: _", + "type_info": "Jsonb" + }, + { + "ordinal": 41, + "name": "ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 42, + "name": "memory_peak", + "type_info": "Int4" + }, + { + "ordinal": 43, + "name": "flow_status: _", + "type_info": "Jsonb" + }, + { + "ordinal": 44, + "name": "flow_leaf_jobs: _", + "type_info": "Jsonb" + }, + { + "ordinal": 45, + "name": "workflow_as_code_status: _", + "type_info": "Jsonb" + }, + { + "ordinal": 46, + "name": "concurrency_key?", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + true, + true, + true, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + false + ] + }, + "hash": "a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4" +} diff --git a/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json b/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json new file mode 100644 index 0000000000..583ae1b55d --- /dev/null +++ b/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json @@ -0,0 +1,127 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job (\n id,\n raw_code,\n raw_lock,\n raw_flow,\n tag,\n workspace_id,\n created_at,\n created_by,\n permissioned_as,\n permissioned_as_email,\n kind,\n runnable_id,\n runnable_path,\n parent_job,\n root_job,\n script_lang,\n script_entrypoint_override,\n flow_step,\n flow_step_id,\n flow_innermost_root_job,\n trigger,\n trigger_kind,\n same_worker,\n visible_to_owner,\n concurrent_limit,\n concurrency_time_window_s,\n cache_ttl,\n timeout,\n priority,\n preprocessed,\n args,\n labels,\n pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10,\n $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30,\n $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Jsonb", + "Varchar", + "Varchar", + "Timestamptz", + "Varchar", + "Varchar", + "Varchar", + { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned" + ] + } + } + }, + "Int8", + "Varchar", + "Uuid", + "Uuid", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + }, + "Varchar", + "Int4", + "Varchar", + "Uuid", + "Varchar", + { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud" + ] + } + } + }, + "Bool", + "Bool", + "Int4", + "Int4", + "Int4", + "Int4", + "Int2", + "Bool", + "Jsonb", + "TextArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b" +} diff --git a/backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json b/backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json new file mode 100644 index 0000000000..ff4479af3f --- /dev/null +++ b/backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job_status (\n id, flow_status, flow_leaf_jobs, workflow_as_code_status\n ) VALUES (\n $1, $2, $3, $4\n )\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d" +} diff --git a/backend/.sqlx/query-cca63828cbb043d2c647830cf50b992d8fa53ec801067e5b6c45daf05e61d136.json b/backend/.sqlx/query-cca63828cbb043d2c647830cf50b992d8fa53ec801067e5b6c45daf05e61d136.json new file mode 100644 index 0000000000..f08ad113b5 --- /dev/null +++ b/backend/.sqlx/query-cca63828cbb043d2c647830cf50b992d8fa53ec801067e5b6c45daf05e61d136.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM job_perms WHERE workspace_id = $1 AND job_id = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "cca63828cbb043d2c647830cf50b992d8fa53ec801067e5b6c45daf05e61d136" +} diff --git a/backend/.sqlx/query-d3038367ca90595eb165bcb6d172239b1cd80d969fa36c9cf3dee0c360dec302.json b/backend/.sqlx/query-d3038367ca90595eb165bcb6d172239b1cd80d969fa36c9cf3dee0c360dec302.json new file mode 100644 index 0000000000..7d948152ec --- /dev/null +++ b/backend/.sqlx/query-d3038367ca90595eb165bcb6d172239b1cd80d969fa36c9cf3dee0c360dec302.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_runtime WHERE id = ANY($1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "d3038367ca90595eb165bcb6d172239b1cd80d969fa36c9cf3dee0c360dec302" +} diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json new file mode 100644 index 0000000000..0e92df7da1 --- /dev/null +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -0,0 +1,411 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n v2_job.id,\n v2_job.raw_code,\n v2_job.raw_lock,\n v2_job.raw_flow as \"raw_flow: _\",\n v2_job.tag,\n v2_job.created_at,\n v2_job.created_by,\n v2_job.permissioned_as,\n v2_job.permissioned_as_email,\n v2_job.kind as \"kind: _\",\n v2_job.runnable_id,\n v2_job.runnable_path,\n v2_job.parent_job,\n v2_job.root_job,\n v2_job.script_lang as \"script_lang: _\",\n v2_job.script_entrypoint_override,\n v2_job.flow_step,\n v2_job.flow_step_id,\n v2_job.flow_innermost_root_job,\n v2_job.trigger,\n v2_job.trigger_kind as \"trigger_kind: _\",\n v2_job.same_worker,\n v2_job.visible_to_owner,\n v2_job.concurrent_limit,\n v2_job.concurrency_time_window_s,\n v2_job.cache_ttl,\n v2_job.timeout,\n v2_job.priority,\n v2_job.preprocessed,\n v2_job.args as \"args: _\",\n v2_job.labels,\n v2_job.pre_run_error,\n v2_job_completed.duration_ms,\n v2_job_completed.result as \"result: _\",\n v2_job_completed.deleted,\n v2_job_completed.canceled_by,\n v2_job_completed.canceled_reason,\n v2_job_completed.flow_status as \"flow_status: _\",\n v2_job_completed.started_at,\n v2_job_completed.memory_peak,\n v2_job_completed.status as \"status: _\",\n v2_job_completed.completed_at,\n v2_job_completed.worker,\n v2_job_completed.workflow_as_code_status as \"workflow_as_code_status: _\",\n v2_job_completed.result_columns,\n v2_job_completed.retries,\n v2_job_completed.extras as \"extras: _\",\n job_logs.logs AS logs,\n job_logs.log_offset,\n job_logs.log_file_index\n FROM v2_job_completed\n INNER JOIN v2_job ON v2_job.id = v2_job_completed.id\n LEFT JOIN v2_job_status ON v2_job_completed.id = v2_job_status.id\n LEFT JOIN job_logs ON job_logs.job_id = v2_job.id\n WHERE v2_job_completed.workspace_id = $1\n ORDER BY v2_job.created_at DESC\n LIMIT $2\n OFFSET $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "raw_code", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "raw_lock", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "raw_flow: _", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "kind: _", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned" + ] + } + } + } + }, + { + "ordinal": 10, + "name": "runnable_id", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 12, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 13, + "name": "root_job", + "type_info": "Uuid" + }, + { + "ordinal": 14, + "name": "script_lang: _", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "script_entrypoint_override", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "flow_step", + "type_info": "Int4" + }, + { + "ordinal": 17, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 18, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 19, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 20, + "name": "trigger_kind: _", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud" + ] + } + } + } + }, + { + "ordinal": 21, + "name": "same_worker", + "type_info": "Bool" + }, + { + "ordinal": 22, + "name": "visible_to_owner", + "type_info": "Bool" + }, + { + "ordinal": 23, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 24, + "name": "concurrency_time_window_s", + "type_info": "Int4" + }, + { + "ordinal": 25, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 26, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 27, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 28, + "name": "preprocessed", + "type_info": "Bool" + }, + { + "ordinal": 29, + "name": "args: _", + "type_info": "Jsonb" + }, + { + "ordinal": 30, + "name": "labels", + "type_info": "TextArray" + }, + { + "ordinal": 31, + "name": "pre_run_error", + "type_info": "Text" + }, + { + "ordinal": 32, + "name": "duration_ms", + "type_info": "Int8" + }, + { + "ordinal": 33, + "name": "result: _", + "type_info": "Jsonb" + }, + { + "ordinal": 34, + "name": "deleted", + "type_info": "Bool" + }, + { + "ordinal": 35, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 36, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 37, + "name": "flow_status: _", + "type_info": "Jsonb" + }, + { + "ordinal": 38, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 39, + "name": "memory_peak", + "type_info": "Int4" + }, + { + "ordinal": 40, + "name": "status: _", + "type_info": { + "Custom": { + "name": "job_status", + "kind": { + "Enum": [ + "success", + "failure", + "canceled", + "skipped" + ] + } + } + } + }, + { + "ordinal": 41, + "name": "completed_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 42, + "name": "worker", + "type_info": "Varchar" + }, + { + "ordinal": 43, + "name": "workflow_as_code_status: _", + "type_info": "Jsonb" + }, + { + "ordinal": 44, + "name": "result_columns", + "type_info": "TextArray" + }, + { + "ordinal": 45, + "name": "retries", + "type_info": "UuidArray" + }, + { + "ordinal": 46, + "name": "extras: _", + "type_info": "Jsonb" + }, + { + "ordinal": 47, + "name": "logs", + "type_info": "Text" + }, + { + "ordinal": 48, + "name": "log_offset", + "type_info": "Int4" + }, + { + "ordinal": 49, + "name": "log_file_index", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + true, + true, + true, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + false, + true, + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + "hash": "e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235" +} diff --git a/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json b/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json new file mode 100644 index 0000000000..c324e19bdf --- /dev/null +++ b/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json @@ -0,0 +1,127 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job (\n id,\n raw_code,\n raw_lock,\n raw_flow,\n tag,\n workspace_id,\n created_at,\n created_by,\n permissioned_as,\n permissioned_as_email,\n kind,\n runnable_id,\n runnable_path,\n parent_job,\n root_job,\n script_lang,\n script_entrypoint_override,\n flow_step,\n flow_step_id,\n flow_innermost_root_job,\n trigger,\n trigger_kind,\n same_worker,\n visible_to_owner,\n concurrent_limit,\n concurrency_time_window_s,\n cache_ttl,\n timeout,\n priority,\n preprocessed,\n args,\n labels,\n pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Jsonb", + "Varchar", + "Varchar", + "Timestamptz", + "Varchar", + "Varchar", + "Varchar", + { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned" + ] + } + } + }, + "Int8", + "Varchar", + "Uuid", + "Uuid", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + }, + "Varchar", + "Int4", + "Varchar", + "Uuid", + "Varchar", + { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud" + ] + } + } + }, + "Bool", + "Bool", + "Int4", + "Int4", + "Int4", + "Int4", + "Int2", + "Bool", + "Jsonb", + "TextArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f" +} diff --git a/backend/.sqlx/query-fae4c8069ef5389e64a35d3867e9080006f1c36a53e1e9e631d794a876d5e0b3.json b/backend/.sqlx/query-fae4c8069ef5389e64a35d3867e9080006f1c36a53e1e9e631d794a876d5e0b3.json new file mode 100644 index 0000000000..8d8fd7f655 --- /dev/null +++ b/backend/.sqlx/query-fae4c8069ef5389e64a35d3867e9080006f1c36a53e1e9e631d794a876d5e0b3.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "fae4c8069ef5389e64a35d3867e9080006f1c36a53e1e9e631d794a876d5e0b3" +} diff --git a/backend/.sqlx/query-fc191f8598ce030fdee04353ad6ca562984f8b86c2882becef42e838bf79375d.json b/backend/.sqlx/query-fc191f8598ce030fdee04353ad6ca562984f8b86c2882becef42e838bf79375d.json new file mode 100644 index 0000000000..2c06b40e6b --- /dev/null +++ b/backend/.sqlx/query-fc191f8598ce030fdee04353ad6ca562984f8b86c2882becef42e838bf79375d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM zombie_job_counter WHERE job_id = ANY($1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "fc191f8598ce030fdee04353ad6ca562984f8b86c2882becef42e838bf79375d" +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 63c20a054b..6dadfa2fb6 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -8841,6 +8841,121 @@ paths: items: $ref: "#/components/schemas/CompletedJob" + /w/{workspace}/jobs/completed/export: + get: + summary: export all completed jobs for backup/migration + operationId: exportCompletedJobs + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + responses: + "200": + description: All completed jobs exported + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ExportableCompletedJob" + + /w/{workspace}/jobs/completed/import: + post: + summary: import completed jobs from backup/migration + operationId: importCompletedJobs + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ExportableCompletedJob" + responses: + "200": + description: Successfully imported completed jobs + content: + text/plain: + schema: + type: string + + /w/{workspace}/jobs/queue/export: + get: + summary: export all queued jobs for backup/migration + operationId: exportQueuedJobs + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + responses: + "200": + description: All queued jobs exported + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ExportableQueuedJob" + + /w/{workspace}/jobs/queue/import: + post: + summary: import queued jobs from backup/migration + operationId: importQueuedJobs + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ExportableQueuedJob" + responses: + "200": + description: Successfully imported queued jobs + content: + text/plain: + schema: + type: string + + /w/{workspace}/jobs/delete: + post: + summary: delete jobs by IDs from all related tables + operationId: deleteJobs + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: array + items: + type: string + format: uuid + description: Array of job IDs to delete + responses: + "200": + description: Successfully deleted jobs + content: + text/plain: + schema: + type: string + description: Summary of deleted jobs and rows + /w/{workspace}/jobs/list: get: summary: list all jobs @@ -16495,6 +16610,297 @@ components: - visible_to_owner - tag + ExportableCompletedJob: + type: object + description: Completed job with full data for export/import operations + properties: + id: + type: string + format: uuid + parent_job: + type: string + format: uuid + created_by: + type: string + created_at: + type: string + format: date-time + started_at: + type: string + format: date-time + completed_at: + type: string + format: date-time + duration_ms: + type: integer + script_path: + type: string + script_hash: + type: string + args: + type: object + description: Full job arguments without size restrictions + result: + type: object + description: Full job result without size restrictions + logs: + type: string + description: Complete job logs from v2_job table + raw_code: + type: string + raw_lock: + type: string + canceled_by: + type: string + canceled_reason: + type: string + job_kind: + type: string + enum: + [ + "script", + "preview", + "dependencies", + "flow", + "flowdependencies", + "appdependencies", + "flowpreview", + "script_hub", + "identity", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + ] + trigger: + type: string + description: Trigger path for the job (replaces schedule_path) + trigger_kind: + type: string + enum: + [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + ] + permissioned_as: + type: string + permissioned_as_email: + type: string + flow_status: + type: object + description: Flow status from v2_job_status table + workflow_as_code_status: + type: object + raw_flow: + type: object + is_flow_step: + type: boolean + language: + $ref: "#/components/schemas/ScriptLang" + is_skipped: + type: boolean + email: + type: string + visible_to_owner: + type: boolean + mem_peak: + type: integer + tag: + type: string + priority: + type: integer + labels: + type: array + items: + type: string + same_worker: + type: boolean + flow_step_id: + type: string + flow_innermost_root_job: + type: string + format: uuid + concurrent_limit: + type: integer + concurrency_time_window_s: + type: integer + timeout: + type: integer + cache_ttl: + type: integer + self_wait_time_ms: + type: integer + aggregate_wait_time_ms: + type: integer + preprocessed: + type: boolean + worker: + type: string + status: + type: string + description: Actual job status from database + required: + - id + - created_by + - created_at + - job_kind + - permissioned_as + - email + - visible_to_owner + + ExportableQueuedJob: + type: object + description: Queued job with full data for export/import operations + properties: + id: + type: string + format: uuid + parent_job: + type: string + format: uuid + created_by: + type: string + created_at: + type: string + format: date-time + started_at: + type: string + format: date-time + scheduled_for: + type: string + format: date-time + script_path: + type: string + script_hash: + type: string + args: + type: object + description: Full job arguments without size restrictions + logs: + type: string + description: Complete job logs from v2_job table + raw_code: + type: string + raw_lock: + type: string + canceled_by: + type: string + canceled_reason: + type: string + job_kind: + type: string + enum: + [ + "script", + "preview", + "dependencies", + "flowdependencies", + "appdependencies", + "flow", + "flowpreview", + "script_hub", + "identity", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + ] + trigger: + type: string + description: Trigger path for the job (replaces schedule_path) + trigger_kind: + type: string + enum: + [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + ] + permissioned_as: + type: string + permissioned_as_email: + type: string + flow_status: + type: object + description: Flow status from v2_job_status table + workflow_as_code_status: + type: object + raw_flow: + type: object + is_flow_step: + type: boolean + language: + $ref: "#/components/schemas/ScriptLang" + email: + type: string + visible_to_owner: + type: boolean + mem_peak: + type: integer + tag: + type: string + priority: + type: integer + labels: + type: array + items: + type: string + same_worker: + type: boolean + flow_step_id: + type: string + flow_innermost_root_job: + type: string + format: uuid + concurrent_limit: + type: integer + concurrency_time_window_s: + type: integer + timeout: + type: integer + cache_ttl: + type: integer + self_wait_time_ms: + type: integer + aggregate_wait_time_ms: + type: integer + preprocessed: + type: boolean + suspend: + type: integer + suspend_until: + type: string + format: date-time + required: + - id + - created_by + - created_at + - job_kind + - permissioned_as + - email + - visible_to_owner + ObscuredJob: type: object properties: diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 584ddad54c..968bf41634 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -260,6 +260,11 @@ pub fn workspaced_service() -> Router { ) .route("/list_filtered_uuids", get(list_filtered_job_uuids)) .route("/queue/list", get(list_queue_jobs)) + .route("/queue/export", get(crate::jobs_export::export_queued_jobs)) + .route( + "/queue/import", + post(crate::jobs_export::import_queued_jobs), + ) .route("/queue/count", get(count_queue_jobs)) .route("/queue/list_filtered_uuids", get(list_filtered_uuids)) .route("/queue/position/:timestamp", get(get_queue_position)) @@ -271,6 +276,18 @@ pub fn workspaced_service() -> Router { "/completed/list", get(list_completed_jobs).layer(cors.clone()), ) + .route( + "/completed/export", + get(crate::jobs_export::export_completed_jobs).layer(cors.clone()), + ) + .route( + "/completed/import", + post(crate::jobs_export::import_completed_jobs).layer(cors.clone()), + ) + .route( + "/delete", + post(crate::jobs_export::delete_jobs), + ) .route( "/completed/get/:id", get(get_completed_job).layer(cors.clone()), diff --git a/backend/windmill-api/src/jobs_export.rs b/backend/windmill-api/src/jobs_export.rs new file mode 100644 index 0000000000..b776096f67 --- /dev/null +++ b/backend/windmill-api/src/jobs_export.rs @@ -0,0 +1,828 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{ + extract::{Path, Query}, + Extension, Json, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use uuid::Uuid; +use windmill_common::{ + db::UserDB, + error, + jobs::{JobKind, JobStatus, JobTriggerKind}, + scripts::ScriptLang, + utils::{paginate, paginate_without_limits, require_admin, Pagination}, +}; + +use crate::db::ApiAuthed; + +#[derive(Serialize, Deserialize)] +pub struct ExportableCompletedJob { + pub id: Uuid, + pub raw_code: Option, + pub raw_lock: Option, + pub raw_flow: Option>>, + pub tag: String, + pub created_at: chrono::DateTime, + pub created_by: String, + pub permissioned_as: String, + pub permissioned_as_email: String, + pub kind: JobKind, + pub runnable_id: Option, + pub runnable_path: Option, + pub parent_job: Option, + pub root_job: Option, + pub script_lang: Option, + pub script_entrypoint_override: Option, + pub flow_step: Option, + pub flow_step_id: Option, + pub flow_innermost_root_job: Option, + pub trigger: Option, + pub trigger_kind: Option, + pub same_worker: bool, + pub visible_to_owner: bool, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub cache_ttl: Option, + pub timeout: Option, + pub priority: Option, + pub preprocessed: Option, + pub args: Option>>, + pub labels: Option>, + pub pre_run_error: Option, + pub duration_ms: i64, + pub result: Option>>, + pub deleted: bool, + pub canceled_by: Option, + pub canceled_reason: Option, + pub flow_status: Option>>, + pub started_at: Option>, + pub memory_peak: Option, + pub status: JobStatus, + pub completed_at: chrono::DateTime, + pub worker: Option, + pub workflow_as_code_status: Option>>, + pub result_columns: Option>, + pub retries: Option>, + pub extras: Option>>, + pub logs: Option, + pub log_offset: Option, + pub log_file_index: Option>, +} + +#[derive(Serialize, Deserialize)] +pub struct ExportableQueuedJob { + // v2_job columns (excluding workspace_id) + pub id: Uuid, + pub raw_code: Option, + pub raw_lock: Option, + pub raw_flow: Option>>, + pub tag: String, + pub created_at: chrono::DateTime, + pub created_by: String, + pub permissioned_as: String, + pub permissioned_as_email: String, + pub kind: JobKind, + pub runnable_id: Option, + pub runnable_path: Option, + pub parent_job: Option, + pub root_job: Option, + pub script_lang: Option, + pub script_entrypoint_override: Option, + pub flow_step: Option, + pub flow_step_id: Option, + pub flow_innermost_root_job: Option, + pub trigger: Option, + pub trigger_kind: Option, + pub same_worker: bool, + pub visible_to_owner: bool, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub cache_ttl: Option, + pub timeout: Option, + pub priority: Option, + pub preprocessed: Option, + pub args: Option>>, + pub labels: Option>, + pub pre_run_error: Option, + + // v2_job_queue columns (excluding workspace_id and id/created_at/tag/priority) + pub started_at: Option>, + pub scheduled_for: chrono::DateTime, + pub running: bool, + pub canceled_by: Option, + pub canceled_reason: Option, + pub suspend: Option, + pub suspend_until: Option>, + pub worker: Option, + pub extras: Option>>, + + // v2_job_runtime columns (excluding id) + pub ping: Option>, + pub memory_peak: Option, + + // v2_job_status columns (excluding id) + pub flow_status: Option>>, + pub flow_leaf_jobs: Option>>, + pub workflow_as_code_status: Option>>, + + // concurrency_key table + pub concurrency_key: Option, +} + +pub async fn export_completed_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, +) -> error::JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + + let (per_page, offset) = paginate(pagination); + let per_page = per_page as i64; + let offset = offset as i64; + + let mut tx = user_db.begin(&authed).await?; + + let jobs = sqlx::query_as!( + ExportableCompletedJob, + r#" + SELECT + v2_job.id, + v2_job.raw_code, + v2_job.raw_lock, + v2_job.raw_flow as "raw_flow: _", + v2_job.tag, + v2_job.created_at, + v2_job.created_by, + v2_job.permissioned_as, + v2_job.permissioned_as_email, + v2_job.kind as "kind: _", + v2_job.runnable_id, + v2_job.runnable_path, + v2_job.parent_job, + v2_job.root_job, + v2_job.script_lang as "script_lang: _", + v2_job.script_entrypoint_override, + v2_job.flow_step, + v2_job.flow_step_id, + v2_job.flow_innermost_root_job, + v2_job.trigger, + v2_job.trigger_kind as "trigger_kind: _", + v2_job.same_worker, + v2_job.visible_to_owner, + v2_job.concurrent_limit, + v2_job.concurrency_time_window_s, + v2_job.cache_ttl, + v2_job.timeout, + v2_job.priority, + v2_job.preprocessed, + v2_job.args as "args: _", + v2_job.labels, + v2_job.pre_run_error, + v2_job_completed.duration_ms, + v2_job_completed.result as "result: _", + v2_job_completed.deleted, + v2_job_completed.canceled_by, + v2_job_completed.canceled_reason, + v2_job_completed.flow_status as "flow_status: _", + v2_job_completed.started_at, + v2_job_completed.memory_peak, + v2_job_completed.status as "status: _", + v2_job_completed.completed_at, + v2_job_completed.worker, + v2_job_completed.workflow_as_code_status as "workflow_as_code_status: _", + v2_job_completed.result_columns, + v2_job_completed.retries, + v2_job_completed.extras as "extras: _", + job_logs.logs AS logs, + job_logs.log_offset, + job_logs.log_file_index + FROM v2_job_completed + INNER JOIN v2_job ON v2_job.id = v2_job_completed.id + LEFT JOIN v2_job_status ON v2_job_completed.id = v2_job_status.id + LEFT JOIN job_logs ON job_logs.job_id = v2_job.id + WHERE v2_job_completed.workspace_id = $1 + ORDER BY v2_job.created_at DESC + LIMIT $2 + OFFSET $3 + "#, + w_id, + per_page, + offset + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(jobs)) +} + +pub async fn export_queued_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, +) -> error::JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + + let (per_page, offset) = paginate_without_limits(pagination); + let per_page = per_page as i64; + let offset = offset as i64; + + let mut tx = user_db.begin(&authed).await?; + + let jobs = sqlx::query_as!( + ExportableQueuedJob, + r#" + SELECT + v2_job.id, + v2_job.raw_code, + v2_job.raw_lock, + v2_job.raw_flow as "raw_flow: _", + v2_job.tag, + v2_job.created_at, + v2_job.created_by, + v2_job.permissioned_as, + v2_job.permissioned_as_email, + v2_job.kind as "kind: _", + v2_job.runnable_id, + v2_job.runnable_path, + v2_job.parent_job, + v2_job.root_job, + v2_job.script_lang as "script_lang: _", + v2_job.script_entrypoint_override, + v2_job.flow_step, + v2_job.flow_step_id, + v2_job.flow_innermost_root_job, + v2_job."trigger", + v2_job.trigger_kind as "trigger_kind: _", + v2_job.same_worker, + v2_job.visible_to_owner, + v2_job.concurrent_limit, + v2_job.concurrency_time_window_s, + v2_job.cache_ttl, + v2_job.timeout, + v2_job.priority, + v2_job.preprocessed, + v2_job.args as "args: _", + v2_job.labels, + v2_job.pre_run_error, + + v2_job_queue.started_at, + v2_job_queue.scheduled_for, + v2_job_queue.running, + v2_job_queue.canceled_by, + v2_job_queue.canceled_reason, + v2_job_queue.suspend, + v2_job_queue.suspend_until, + v2_job_queue.worker, + v2_job_queue.extras as "extras: _", + + v2_job_runtime.ping, + v2_job_runtime.memory_peak, + + v2_job_status.flow_status as "flow_status: _", + v2_job_status.flow_leaf_jobs as "flow_leaf_jobs: _", + v2_job_status.workflow_as_code_status as "workflow_as_code_status: _", + + concurrency_key.key as "concurrency_key?" + FROM v2_job_queue + INNER JOIN v2_job ON v2_job.id = v2_job_queue.id + LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id + LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id + LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id + WHERE v2_job_queue.workspace_id = $1 + AND v2_job_queue.running = false + AND v2_job.parent_job IS NULL + AND v2_job.trigger_kind IS DISTINCT FROM 'schedule' + ORDER BY v2_job.created_at DESC + LIMIT $2 + OFFSET $3 + "#, + w_id, + per_page, + offset + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(jobs)) +} + +pub async fn import_completed_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(jobs): Json>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = user_db.begin(&authed).await?; + + for job in jobs { + // Insert into v2_job table + sqlx::query!( + r#" + INSERT INTO v2_job ( + id, + raw_code, + raw_lock, + raw_flow, + tag, + workspace_id, + created_at, + created_by, + permissioned_as, + permissioned_as_email, + kind, + runnable_id, + runnable_path, + parent_job, + root_job, + script_lang, + script_entrypoint_override, + flow_step, + flow_step_id, + flow_innermost_root_job, + trigger, + trigger_kind, + same_worker, + visible_to_owner, + concurrent_limit, + concurrency_time_window_s, + cache_ttl, + timeout, + priority, + preprocessed, + args, + labels, + pre_run_error + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, + $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + job.raw_code, + job.raw_lock, + job.raw_flow as _, + &job.tag, + &w_id, + job.created_at, + &job.created_by, + &job.permissioned_as, + job.permissioned_as_email, + job.kind as _, + job.runnable_id, + job.runnable_path, + job.parent_job, + job.root_job, + job.script_lang as _, + job.script_entrypoint_override, + job.flow_step, + job.flow_step_id, + job.flow_innermost_root_job, + job.trigger, + job.trigger_kind as _, + job.same_worker, + job.visible_to_owner, + job.concurrent_limit, + job.concurrency_time_window_s, + job.cache_ttl, + job.timeout, + job.priority, + job.preprocessed, + job.args as _, + job.labels as _, + job.pre_run_error + ) + .execute(&mut *tx) + .await?; + + // Insert into v2_job_completed table + sqlx::query!( + r#" + INSERT INTO v2_job_completed ( + id, + workspace_id, + started_at, + completed_at, + duration_ms, + result, + deleted, + canceled_by, + canceled_reason, + flow_status, + memory_peak, + status, + worker, + workflow_as_code_status, + result_columns, + retries, + extras + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + &w_id, + job.started_at, + job.completed_at, + job.duration_ms, + job.result as _, + job.deleted, + job.canceled_by, + job.canceled_reason, + job.flow_status as _, + job.memory_peak, + job.status as _, + job.worker, + job.workflow_as_code_status as _, + job.result_columns as _, + job.retries as _, + job.extras as _ + ) + .execute(&mut *tx) + .await?; + + // Insert into job_logs table + if let Some(logs) = &job.logs { + sqlx::query!( + r#" + INSERT INTO job_logs ( + job_id, + workspace_id, + logs, + log_offset, + log_file_index + ) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (job_id) DO NOTHING + "#, + job.id, + &w_id, + logs, + job.log_offset, + job.log_file_index as _ + ) + .execute(&mut *tx) + .await?; + } + } + + tx.commit().await?; + Ok(format!("Successfully imported jobs")) +} + +pub async fn import_queued_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(jobs): Json>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = user_db.begin(&authed).await?; + + for job in jobs { + // Insert into v2_job table + // Insert into v2_job table, fields and order adapted to match ExportableQueuedJob + sqlx::query!( + r#" + INSERT INTO v2_job ( + id, + raw_code, + raw_lock, + raw_flow, + tag, + workspace_id, + created_at, + created_by, + permissioned_as, + permissioned_as_email, + kind, + runnable_id, + runnable_path, + parent_job, + root_job, + script_lang, + script_entrypoint_override, + flow_step, + flow_step_id, + flow_innermost_root_job, + trigger, + trigger_kind, + same_worker, + visible_to_owner, + concurrent_limit, + concurrency_time_window_s, + cache_ttl, + timeout, + priority, + preprocessed, + args, + labels, + pre_run_error + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, + $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, + $31, $32, $33 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + job.raw_code, + job.raw_lock, + job.raw_flow as _, + &job.tag, + &w_id, + job.created_at, + &job.created_by, + &job.permissioned_as, + &job.permissioned_as_email, + job.kind as _, + job.runnable_id, + job.runnable_path, + job.parent_job, + job.root_job, + job.script_lang as _, + job.script_entrypoint_override, + job.flow_step, + job.flow_step_id, + job.flow_innermost_root_job, + job.trigger, + job.trigger_kind as _, + job.same_worker, + job.visible_to_owner, + job.concurrent_limit, + job.concurrency_time_window_s, + job.cache_ttl, + job.timeout, + job.priority, + job.preprocessed, + job.args as _, + job.labels as _, + job.pre_run_error, + ) + .execute(&mut *tx) + .await?; + + // Insert into v2_job_queue table + sqlx::query!( + r#" + INSERT INTO v2_job_queue ( + id, + workspace_id, + started_at, + scheduled_for, + running, + canceled_by, + canceled_reason, + suspend, + suspend_until, + worker, + extras, + tag, + priority + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + &w_id, + job.started_at, + job.scheduled_for, + job.running, + job.canceled_by, + job.canceled_reason, + job.suspend, + job.suspend_until, + job.worker, + job.extras as _, + &job.tag, + job.priority + ) + .execute(&mut *tx) + .await?; + + // Insert into v2_job_runtime if any present + if job.ping.is_some() || job.memory_peak.is_some() { + sqlx::query!( + r#" + INSERT INTO v2_job_runtime ( + id, ping, memory_peak + ) VALUES ( + $1, $2, $3 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + job.ping, + job.memory_peak + ) + .execute(&mut *tx) + .await?; + } + + // Insert into v2_job_status if present + if job.flow_status.is_some() + || job.flow_leaf_jobs.is_some() + || job.workflow_as_code_status.is_some() + { + sqlx::query!( + r#" + INSERT INTO v2_job_status ( + id, flow_status, flow_leaf_jobs, workflow_as_code_status + ) VALUES ( + $1, $2, $3, $4 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + job.flow_status as _, + job.flow_leaf_jobs as _, + job.workflow_as_code_status as _ + ) + .execute(&mut *tx) + .await?; + } + + // Insert concurrency_key if present + if let Some(ref concurrency_key) = job.concurrency_key { + sqlx::query!( + r#" + WITH inserted_concurrency_counter AS ( + INSERT INTO concurrency_counter (concurrency_id, job_uuids) + VALUES ($1, '{}'::jsonb) + ON CONFLICT DO NOTHING + ) + INSERT INTO concurrency_key(key, job_id) + VALUES ($1, $2) + ON CONFLICT (job_id) DO NOTHING + "#, + concurrency_key, + job.id + ) + .execute(&mut *tx) + .await?; + } + } + + tx.commit().await?; + tracing::info!("Successfully imported jobs"); + Ok(format!("Successfully imported jobs")) +} + +pub async fn delete_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(job_ids): Json>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + + if job_ids.is_empty() { + return Ok("No jobs to delete".to_string()); + } + + let mut tx = user_db.begin(&authed).await?; + + // Delete from all job-related tables + // Order matters due to foreign key constraints + + // 1. Delete job logs + let logs_deleted = sqlx::query!( + "DELETE FROM job_logs WHERE workspace_id = $1 AND job_id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + // 2. Delete job permissions + let perms_deleted = sqlx::query!( + "DELETE FROM job_perms WHERE workspace_id = $1 AND job_id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + // 3. Delete job stats + let stats_deleted = sqlx::query!( + "DELETE FROM job_stats WHERE workspace_id = $1 AND job_id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + // 4. Delete resume jobs + let resume_deleted = sqlx::query!("DELETE FROM resume_job WHERE job = ANY($1)", &job_ids) + .execute(&mut *tx) + .await? + .rows_affected(); + + // 5. Delete from v2_job_runtime + let runtime_deleted = sqlx::query!("DELETE FROM v2_job_runtime WHERE id = ANY($1)", &job_ids) + .execute(&mut *tx) + .await? + .rows_affected(); + + // 6. Delete from v2_job_status + let status_deleted = sqlx::query!("DELETE FROM v2_job_status WHERE id = ANY($1)", &job_ids) + .execute(&mut *tx) + .await? + .rows_affected(); + + // 7. Delete from concurrency_key + let concurrency_key_deleted = sqlx::query!( + "DELETE FROM concurrency_key WHERE job_id = ANY($1)", + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + // 8. Delete from v2_job_queue (for queued jobs) + let queue_deleted = sqlx::query!( + "DELETE FROM v2_job_queue WHERE workspace_id = $1 AND id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + // 9. Delete from v2_job_completed (for completed jobs) + let completed_deleted = sqlx::query!( + "DELETE FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + // 10. Delete zombie job counter + let zombie_deleted = sqlx::query!( + "DELETE FROM zombie_job_counter WHERE job_id = ANY($1)", + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + // 11. Finally, delete from v2_job (main table) + let jobs_deleted = sqlx::query!( + "DELETE FROM v2_job WHERE workspace_id = $1 AND id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + tx.commit().await?; + + let total_rows_deleted = logs_deleted + + perms_deleted + + stats_deleted + + resume_deleted + + runtime_deleted + + status_deleted + + concurrency_key_deleted + + queue_deleted + + completed_deleted + + zombie_deleted + + jobs_deleted; + + tracing::info!( + "Successfully deleted {} jobs ({} total rows across all tables) from workspace {}", + job_ids.len(), + total_rows_deleted, + w_id + ); + + Ok(format!( + "Successfully deleted {} jobs ({} total rows across all tables)", + jobs_deleted, total_rows_deleted + )) +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index ac39df0ee2..0e69924e89 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -127,6 +127,7 @@ pub mod job_helpers_ee; mod job_helpers_oss; pub mod job_metrics; pub mod jobs; +pub mod jobs_export; #[cfg(all(feature = "oauth2", feature = "private"))] pub mod oauth2_ee; #[cfg(feature = "oauth2")] diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index d4589dd5fd..7cd332cb7e 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -96,6 +96,16 @@ pub enum JobKind { AIAgent, } +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)] +#[sqlx(type_name = "JOB_STATUS", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum JobStatus { + Success, + Failure, + Canceled, + Skipped, +} + impl JobKind { pub fn is_flow(&self) -> bool { matches!( diff --git a/cli/src/commands/jobs/jobs.ts b/cli/src/commands/jobs/jobs.ts new file mode 100644 index 0000000000..f7dadfdb26 --- /dev/null +++ b/cli/src/commands/jobs/jobs.ts @@ -0,0 +1,286 @@ +// deno-lint-ignore-file no-explicit-any +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { colors, Command, Confirm, log } from "../../../deps.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import * as fs from "node:fs/promises"; +import * as wmill from "../../../gen/services.gen.ts"; + +async function pullJobs( + opts: GlobalOptions & { + completedOutput?: string; + queuedOutput?: string; + skipWorkerCheck?: boolean; + }, + workspace?: string +) { + opts = await mergeConfigWithConfigFile(opts); + const ws = await resolveWorkspace({ ...opts, workspace }); + await requireLogin(opts); + + log.info("Pulling jobs from workspace " + ws.workspaceId); + + // Check for active workers and warn user + if (!opts.skipWorkerCheck) { + try { + const workers = await wmill.listWorkers({ pingSince: 60 }); + + if (workers.length > 0) { + log.info( + colors.yellow( + `\nWarning: Found ${workers.length} active worker(s) on the instance.` + ) + ); + log.info( + "It's recommended to scale down all workers before exporting jobs to ensure that no new jobs are being created during export." + ); + + const proceed = await Confirm.prompt({ + message: "Do you want to continue with the export anyway?", + default: false, + }); + + if (!proceed) { + log.info( + "Export cancelled. Please scale down workers and try again." + ); + log.info("You can skip this check with --skip-worker-check flag."); + return; + } + } + } catch (e) { + log.debug(`Could not check for active workers: ${e}`); + } + } + + // Pull completed jobs + let completedJobs: any[] = []; + let page = 1; + const perPage = 1000; + + while (true) { + const batch = await wmill.exportCompletedJobs({ + workspace: ws.workspaceId, + page, + perPage, + }); + + if (batch.length === 0) break; + completedJobs = completedJobs.concat(batch); + + if (batch.length < perPage) break; + page++; + } + + const completedPath = opts.completedOutput || "completed_jobs.json"; + await fs.writeFile(completedPath, JSON.stringify(completedJobs, null, 2)); + log.info( + colors.green( + `Successfully pulled ${completedJobs.length} completed jobs to ${completedPath}` + ) + ); + + // Pull queued jobs + let queuedJobs: any[] = []; + page = 1; + + while (true) { + const batch = await wmill.exportQueuedJobs({ + workspace: ws.workspaceId, + page, + perPage, + }); + + if (batch.length === 0) break; + queuedJobs = queuedJobs.concat(batch); + + if (batch.length < perPage) break; + page++; + } + const queuedPath = opts.queuedOutput || "queued_jobs.json"; + await fs.writeFile(queuedPath, JSON.stringify(queuedJobs, null, 2)); + log.info( + colors.green( + `Successfully pulled ${queuedJobs.length} queued jobs to ${queuedPath}` + ) + ); + + // Ask to delete all jobs (queued and completed) + const allJobs = [...queuedJobs, ...completedJobs]; + if (allJobs.length > 0) { + const confirmed = await Confirm.prompt({ + message: `Do you want to delete the ${allJobs.length} pulled jobs (queued + completed) from the workspace? If you don't, you won't be able to import them again on the same instance.`, + default: false, + }); + + if (confirmed) { + const jobIds = allJobs.map((job: any) => job.id); + + await wmill.deleteJobs({ + workspace: ws.workspaceId, + requestBody: jobIds, + }); + + log.info( + colors.green(`Deleted ${jobIds.length} jobs (queued + completed)`) + ); + } else { + log.info("Skipping deletion of jobs"); + } + } +} + +async function pushJobs( + opts: GlobalOptions & { + completedFile?: string; + queuedFile?: string; + skipWorkerCheck?: boolean; + }, + workspace?: string +) { + opts = await mergeConfigWithConfigFile(opts); + const ws = await resolveWorkspace({ ...opts, workspace }); + await requireLogin(opts); + + log.info(`Pushing jobs to workspace ${ws.workspaceId}`); + + // Check for active workers before importing + if (!opts.skipWorkerCheck) { + try { + const workers = await wmill.listWorkers({ pingSince: 60 }); + + if (workers.length > 0) { + log.info( + colors.yellow( + `\nWarning: Found ${workers.length} active worker(s) on the instance.` + ) + ); + log.info( + "It's recommended to scale down all workers before importing jobs to ensure:" + ); + log.info( + " - No imported jobs are processed immediately during import" + ); + log.info( + " - You have time to review or adjust the imported jobs before they start running" + ); + log.info(""); + + const proceed = await Confirm.prompt({ + message: "Do you want to continue with the import anyway?", + default: false, + }); + + if (!proceed) { + log.info( + "Import cancelled. Please scale down workers and try again." + ); + log.info("You can skip this check with --skip-worker-check flag."); + return; + } + } + } catch (e) { + log.debug(`Could not check for active workers: ${e}`); + } + } + + // Push completed jobs + const completedPath = opts.completedFile || "completed_jobs.json"; + try { + const completedContent = await fs.readFile(completedPath, "utf-8"); + const completedJobs = JSON.parse(completedContent); + + if (!Array.isArray(completedJobs)) { + throw new Error("Completed jobs file must contain an array of jobs"); + } + + const completedResult = await wmill.importCompletedJobs({ + workspace: ws.workspaceId, + requestBody: completedJobs, + }); + + log.info(colors.green(`Completed jobs: ${completedResult}`)); + } catch (e: any) { + if (e.code === "ENOENT") { + log.info( + colors.yellow( + `No completed jobs file found at ${completedPath}, skipping` + ) + ); + } else { + throw new Error(`Failed to push completed jobs: ${e}`); + } + } + + // Push queued jobs + const queuedPath = opts.queuedFile || "queued_jobs.json"; + try { + const queuedContent = await fs.readFile(queuedPath, "utf-8"); + const queuedJobs = JSON.parse(queuedContent); + + if (!Array.isArray(queuedJobs)) { + throw new Error("Queued jobs file must contain an array of jobs"); + } + + const queuedResult = await wmill.importQueuedJobs({ + workspace: ws.workspaceId, + requestBody: queuedJobs, + }); + + log.info(colors.green(`Queued jobs: ${queuedResult}`)); + } catch (e: any) { + if (e.code === "ENOENT") { + log.info( + colors.yellow(`No queued jobs file found at ${queuedPath}, skipping`) + ); + } else { + throw new Error(`Failed to push queued jobs: ${e}`); + } + } +} + +const pull = new Command() + .description("Pull completed and queued jobs from workspace") + .option( + "-c, --completed-output ", + "Completed jobs output file (default: completed_jobs.json)" + ) + .option( + "-q, --queued-output ", + "Queued jobs output file (default: queued_jobs.json)" + ) + .option( + "-d, --delete-queued", + "Prompt to delete queued jobs from workspace after pulling" + ) + .option( + "--skip-worker-check", + "Skip checking for active workers before export" + ) + .arguments("[workspace:string]") + .action(pullJobs as any); + +const push = new Command() + .description("Push completed and queued jobs to workspace") + .option( + "-c, --completed-file ", + "Completed jobs input file (default: completed_jobs.json)" + ) + .option( + "-q, --queued-file ", + "Queued jobs input file (default: queued_jobs.json)" + ) + .option( + "--skip-worker-check", + "Skip checking for active workers before import" + ) + .arguments("[workspace:string]") + .action(pushJobs as any); + +const command = new Command() + .description("Manage jobs (import/export)") + .command("pull", pull) + .command("push", push); + +export default command; diff --git a/cli/src/main.ts b/cli/src/main.ts index 2530ff12f0..c79f5c1012 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -38,6 +38,7 @@ import workers from "./commands/workers/workers.ts"; import queues from "./commands/queues/queues.ts"; import dependencies from "./commands/dependencies/dependencies.ts"; import init from "./commands/init/init.ts"; +import jobs from "./commands/jobs/jobs.ts"; export { flow, @@ -128,6 +129,7 @@ const command = new Command() .command("workers", workers) .command("queues", queues) .command("dependencies", dependencies) + .command("jobs", jobs) .command("version --version", "Show version information") .action(async (opts: any) => { console.log("CLI version: " + VERSION); From b27018f3acac01682922b289f9ec21a0b0f3f95f Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Mon, 1 Dec 2025 15:16:22 +0100 Subject: [PATCH 02/10] nits jobs migration command --- backend/windmill-api/src/jobs_export.rs | 1 - cli/src/commands/jobs/jobs.ts | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/backend/windmill-api/src/jobs_export.rs b/backend/windmill-api/src/jobs_export.rs index b776096f67..f1b83cba05 100644 --- a/backend/windmill-api/src/jobs_export.rs +++ b/backend/windmill-api/src/jobs_export.rs @@ -683,7 +683,6 @@ pub async fn import_queued_jobs( } tx.commit().await?; - tracing::info!("Successfully imported jobs"); Ok(format!("Successfully imported jobs")) } diff --git a/cli/src/commands/jobs/jobs.ts b/cli/src/commands/jobs/jobs.ts index f7dadfdb26..495e513c87 100644 --- a/cli/src/commands/jobs/jobs.ts +++ b/cli/src/commands/jobs/jobs.ts @@ -110,7 +110,7 @@ async function pullJobs( const allJobs = [...queuedJobs, ...completedJobs]; if (allJobs.length > 0) { const confirmed = await Confirm.prompt({ - message: `Do you want to delete the ${allJobs.length} pulled jobs (queued + completed) from the workspace? If you don't, you won't be able to import them again on the same instance.`, + message: `Do you want to delete the ${allJobs.length} pulled jobs (queued + completed) from the workspace? If you don't, you won't be able to import them again on the same instance because of the unique constraint on the job ID.`, default: false, }); @@ -250,10 +250,6 @@ const pull = new Command() "-q, --queued-output ", "Queued jobs output file (default: queued_jobs.json)" ) - .option( - "-d, --delete-queued", - "Prompt to delete queued jobs from workspace after pulling" - ) .option( "--skip-worker-check", "Skip checking for active workers before export" From 16ef690cace19852a20a17d65f4a78c67e1942b3 Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:21:43 +0100 Subject: [PATCH 03/10] nit: do not use wildcard for clone_script (#7269) * nit: do not use wildcard for clone_script Signed-off-by: pyranota * Update SQLx metadata * reorder fields in query Signed-off-by: pyranota * Update SQLx metadata * test: make is_template optional Signed-off-by: pyranota * Update SQLx metadata * gave on using macro Signed-off-by: pyranota * Update SQLx metadata * remove dbg! Signed-off-by: pyranota --------- Signed-off-by: pyranota Co-authored-by: windmill-internal-app[bot] --- ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...e33244fe8ab9a803248fc23f334034d24aad4.json | 6 +- ...be955155b7d0344dfa908451faf2a0ba9269b.json | 6 +- ...9352d4d1e3cb3a8a08f1c9c03055a1cab1235.json | 6 +- ...b691ef7d1b0ad9eb2596ccfa7227721a0784f.json | 6 +- backend/windmill-common/src/scripts.rs | 75 +++++++++++++++---- 6 files changed, 70 insertions(+), 31 deletions(-) diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json index c3c57b081c..3a843d3ab3 100644 --- a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -74,8 +74,7 @@ "flowscript", "flownode", "appscript", - "aiagent", - "unassigned" + "aiagent" ] } } @@ -182,8 +181,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json b/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json index 583ae1b55d..73ad47bc5c 100644 --- a/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json +++ b/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json @@ -38,8 +38,7 @@ "flowscript", "flownode", "appscript", - "aiagent", - "unassigned" + "aiagent" ] } } @@ -102,8 +101,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json index 0e92df7da1..3319cce7ff 100644 --- a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -74,8 +74,7 @@ "flowscript", "flownode", "appscript", - "aiagent", - "unassigned" + "aiagent" ] } } @@ -182,8 +181,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json b/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json index c324e19bdf..be88c36d08 100644 --- a/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json +++ b/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json @@ -38,8 +38,7 @@ "flowscript", "flownode", "appscript", - "aiagent", - "unassigned" + "aiagent" ] } } @@ -102,8 +101,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index ca34b5eb5e..191fe7343a 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -231,7 +231,7 @@ impl From for ScriptHash { } } -#[derive(PartialEq, sqlx::Type)] +#[derive(PartialEq, sqlx::Type, Debug)] #[sqlx(transparent, no_pg_array)] pub struct ScriptHashes(pub Vec); @@ -347,7 +347,7 @@ pub fn id_to_codebase_info(id: &str) -> CodebaseInfo { let is_esm = id.contains(".esm"); CodebaseInfo { is_tar, is_esm } } -#[derive(Serialize, sqlx::FromRow)] +#[derive(Serialize, sqlx::FromRow, Debug)] pub struct Script { pub workspace_id: String, pub hash: ScriptHash, @@ -361,7 +361,8 @@ pub struct Script { pub archived: bool, pub schema: Option, pub deleted: bool, - pub is_template: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub is_template: Option, pub extra_perms: serde_json::Value, #[serde(skip_serializing_if = "Option::is_none")] pub lock: Option, @@ -815,6 +816,61 @@ pub fn hash_script(ns: &NewScript) -> i64 { dh.finish() as i64 } +pub async fn fetch_script_for_update<'a>( + hash: ScriptHash, + w_id: &str, + e: impl sqlx::Executor<'a, Database = sqlx::Postgres>, +) -> crate::error::Result> { + sqlx::query_as::<_, Script>( + "SELECT + workspace_id, + hash, + path, + parent_hashes, + summary, + description, + content, + created_by, + created_at, + archived, + schema, + deleted, + is_template, + extra_perms, + lock, + lock_error_logs, + language, + kind, + tag, + draft_only, + envs, + concurrency_key, + concurrent_limit, + concurrency_time_window_s, + debounce_key, + debounce_delay_s, + dedicated_worker, + ws_error_handler_muted, + priority, + cache_ttl, + timeout, + delete_after_use, + restart_unless_cancelled, + visible_to_runner_only, + no_main_func, + codebase, + has_preprocessor, + on_behalf_of_email, + assets + FROM script WHERE hash = $1 AND workspace_id = $2 AND archived = false FOR UPDATE", + ) + .bind(hash.0) + .bind(w_id) + .fetch_optional(e) + .await + .map_err(crate::error::Error::from) +} + pub struct ClonedScript { pub old_script: NewScript, pub new_hash: i64, @@ -826,16 +882,7 @@ pub async fn clone_script<'c>( deployment_message: Option, tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, ) -> crate::error::Result { - // TODO:! - let s = sqlx::query_as::<_, Script>( - "SELECT * FROM script WHERE hash = $1 AND workspace_id = $2 AND archived = false FOR UPDATE", - ) - .bind(base_hash.0) - .bind(w_id) - .fetch_optional(&mut **tx) - .await?; - - let s = if let Some(s) = s { + let s = if let Some(s) = fetch_script_for_update(base_hash, w_id, &mut **tx).await? { s } else { return Err(crate::error::Error::NotFound(format!( @@ -851,7 +898,7 @@ pub async fn clone_script<'c>( description: s.description, content: s.content, schema: s.schema, - is_template: Some(s.is_template), + is_template: s.is_template, // TODO: Make it either None everywhere (particularly when raw reqs are calculated) // Or handle this case and conditionally make Some (only with raw reqs) lock: None, From 6dc22c547f66ccefa12500c70650b44662adbbe9 Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Tue, 2 Dec 2025 11:17:37 +0100 Subject: [PATCH 04/10] nit(cli): remove `--use-raw-requirements` (#7272) new workspace dependencies system renders it useless Signed-off-by: pyranota --- cli/src/commands/flow/flow.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index d3475f9cd8..954b09f876 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -196,7 +196,6 @@ async function run( async function generateLocks( opts: GlobalOptions & { yes?: boolean; - useRawRequirements?: boolean; } & SyncOptions, folder: string | undefined ) { @@ -322,10 +321,6 @@ const command = new Command() ) .arguments("[flow:file]") .option("--yes", "Skip confirmation prompt") - .option( - "-r --use-raw-requirements", - "Use raw requirements (requirements.txt, go.mod, package.json, etc) instead of generating them on the server (can also be set with USE_RAW_REQUIREMENTS=true environment variable)" - ) .option( "-i --includes ", "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)" From 67e30c7741a0391e57e30fbc41c1faa731753dcb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 2 Dec 2025 12:28:50 +0000 Subject: [PATCH 05/10] fix(cli): handle better public apps in legacy mode --- cli/src/commands/app/apps.ts | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/cli/src/commands/app/apps.ts b/cli/src/commands/app/apps.ts index 08fca6c352..8255532bee 100644 --- a/cli/src/commands/app/apps.ts +++ b/cli/src/commands/app/apps.ts @@ -30,22 +30,26 @@ function respecializeFields(fields: Record) { Object.entries(fields).forEach(([k, v]) => { if (typeof v == "object") { if (v.value !== undefined) { - fields[k] = { value: v.value, type: "static" } + fields[k] = { value: v.value, type: "static" }; } else if (v.expr !== undefined) { - fields[k] = { expr: v.expr, allowUserResources: v.allowUserResources, type: "javascript" } + fields[k] = { + expr: v.expr, + allowUserResources: v.allowUserResources, + type: "javascript", + }; } } - }) + }); } export function repopulateFields(runnables: Record) { Object.values(runnables).forEach((v) => { if (typeof v == "object") { if (v.fields !== undefined) { - respecializeFields(v.fields) + respecializeFields(v.fields); } } - }) + }); } export function replaceInlineScripts(rec: any, localPath: string) { if (!rec) { @@ -53,11 +57,10 @@ export function replaceInlineScripts(rec: any, localPath: string) { } if (typeof rec == "object") { return Object.entries(rec).flatMap(([k, v]) => { - if (k == 'runType') { - rec["type"] = 'path' - + if (k == "runType") { + rec["type"] = "path"; } else if (k == "inlineScript" && typeof v == "object") { - rec["type"] = 'inline' + rec["type"] = "inline"; const o: Record = v as any; if (o["content"] && o["content"].startsWith("!inline")) { @@ -114,7 +117,12 @@ export async function pushApp( const localApp = (await yamlParseFile(path)) as AppFile; replaceInlineScripts(localApp.value, localPath); - await generatingPolicy(localApp, remotePath, localApp?.["public"] ?? false); + await generatingPolicy( + localApp, + remotePath, + localApp?.["public"] ?? + localApp?.["policy"]?.["execution_mode"] == "anonymous" + ); if (app) { if (isSuperset(localApp, app)) { log.info(colors.green(`App ${remotePath} is up to date`)); From dc9d25289b7282b0f0bfdf987d8e62e3cd7ea8c6 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 2 Dec 2025 12:47:34 +0000 Subject: [PATCH 06/10] fix(cli): handle better apps pushing back-compatibility --- cli/src/commands/app/apps.ts | 9 +- cli/src/commands/sync/global.ts | 31 +++++ cli/src/commands/sync/sync.ts | 206 +++++++++++++++++++++----------- 3 files changed, 173 insertions(+), 73 deletions(-) create mode 100644 cli/src/commands/sync/global.ts diff --git a/cli/src/commands/app/apps.ts b/cli/src/commands/app/apps.ts index 8255532bee..91933c0f02 100644 --- a/cli/src/commands/app/apps.ts +++ b/cli/src/commands/app/apps.ts @@ -16,6 +16,7 @@ import { ListableApp, Policy } from "../../../gen/types.gen.ts"; import { GlobalOptions, isSuperset } from "../../types.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; import devCommand from "./dev.ts"; +import { isVersionsGeq15851 } from "../sync/global.ts"; export interface AppFile { value: any; @@ -58,9 +59,13 @@ export function replaceInlineScripts(rec: any, localPath: string) { if (typeof rec == "object") { return Object.entries(rec).flatMap(([k, v]) => { if (k == "runType") { - rec["type"] = "path"; + if (isVersionsGeq15851()) { + rec["type"] = "path"; + } } else if (k == "inlineScript" && typeof v == "object") { - rec["type"] = "inline"; + if (isVersionsGeq15851()) { + rec["type"] = "inline"; + } const o: Record = v as any; if (o["content"] && o["content"].startsWith("!inline")) { diff --git a/cli/src/commands/sync/global.ts b/cli/src/commands/sync/global.ts new file mode 100644 index 0000000000..bd64e1452e --- /dev/null +++ b/cli/src/commands/sync/global.ts @@ -0,0 +1,31 @@ +import { colors, log } from "../../../deps.ts"; + +let GLOBAL_VERSIONS: { + remoteMajor: number | undefined; + remoteMinor: number | undefined; +} = { + remoteMajor: undefined, + remoteMinor: undefined, +}; + +export function updateGlobalVersions(version: string) { + try { + const [prefix, remoteMinorStr] = version.split("."); + + GLOBAL_VERSIONS = { + remoteMajor: parseInt(prefix.split("v")[1]), + remoteMinor: parseInt(remoteMinorStr), + }; + } catch (e) { + log.info(colors.gray(`Error reading remote version: ${e}`)); + } +} + +export function isVersionsGeq15851(): boolean { + return ( + GLOBAL_VERSIONS.remoteMajor !== undefined && + GLOBAL_VERSIONS.remoteMajor >= 1 && + GLOBAL_VERSIONS.remoteMinor !== undefined && + GLOBAL_VERSIONS.remoteMinor >= 5851 + ); +} diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 4282b42298..8fffa5ed32 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -34,7 +34,12 @@ import { } from "../script/script.ts"; import { handleFile } from "../script/script.ts"; -import { deepEqual, isFileResource, isRawAppFile, isWorkspaceDependencies } from "../../utils/utils.ts"; +import { + deepEqual, + isFileResource, + isRawAppFile, + isWorkspaceDependencies, +} from "../../utils/utils.ts"; import { getEffectiveSettings, mergeConfigWithConfigFile, @@ -70,6 +75,7 @@ import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../w import { generateFlowLockInternal } from "../flow/flow_metadata.ts"; import { isExecutionModeAnonymous } from "../app/apps.ts"; import { generateAppLocksInternal } from "../app/app_metadata.ts"; +import { updateGlobalVersions } from "./global.ts"; // Merge CLI options with effective settings, preserving CLI flags as overrides function mergeCliWithEffectiveOptions< @@ -268,29 +274,28 @@ function extractFields(fields: Record) { Object.entries(fields).forEach(([k, v]) => { if (typeof v == "object") { if (v.type == "static") { - fields[k] = { value: v.value } + fields[k] = { value: v.value }; } else if (v.type == "javascript") { - fields[k] = { expr: v.expr, allowUserResources: v.allowUserResources } + fields[k] = { expr: v.expr, allowUserResources: v.allowUserResources }; } else if (v.type == "user") { - fields[k] = undefined + fields[k] = undefined; } } // if (k == 'runType') { // fields["type"] = undefined // fields["schema"] = undefined // } - }) + }); } - export function extractFieldsForRawApps(runnables: Record) { Object.values(runnables).forEach((v) => { if (typeof v == "object") { if (v.fields !== undefined) { - extractFields(v.fields) + extractFields(v.fields); } } - }) + }); } export function extractInlineScriptsForApps( key: string | undefined, @@ -303,12 +308,12 @@ export function extractInlineScriptsForApps( } if (typeof rec == "object") { return Object.entries(rec).flatMap(([k, v]) => { - if (k == 'runType') { - rec["type"] = undefined - rec["schema"] = undefined - return [] + if (k == "runType") { + rec["type"] = undefined; + rec["schema"] = undefined; + return []; } else if (k == "inlineScript" && typeof v == "object") { - rec["type"] = undefined + rec["type"] = undefined; const o: Record = v as any; const name = toId(key ?? "", rec); const [basePathO, ext] = pathAssigner.assignPath(name, o["language"]); @@ -351,20 +356,26 @@ function ZipFSElement( p: string, f: JSZip.JSZipObject ): Promise { - const kind: "flow" | "app" | "script" | "resource" | "other" | "raw_app" | "dependencies" = - p.endsWith(".flow.json") - ? "flow" - : p.endsWith(".app.json") - ? "app" - : p.endsWith(".raw_app.json") - ? "raw_app" - : p.endsWith(".script.json") - ? "script" - : p.endsWith(".resource.json") - ? "resource" - : p.startsWith("dependencies/") + const kind: + | "flow" + | "app" + | "script" + | "resource" + | "other" + | "raw_app" + | "dependencies" = p.endsWith(".flow.json") + ? "flow" + : p.endsWith(".app.json") + ? "app" + : p.endsWith(".raw_app.json") + ? "raw_app" + : p.endsWith(".script.json") + ? "script" + : p.endsWith(".resource.json") + ? "resource" + : p.startsWith("dependencies/") ? "dependencies" - : "other"; + : "other"; const isJson = p.endsWith(".json"); @@ -502,7 +513,7 @@ function ZipFSElement( undefined, value, newPathAssigner(defaultTs), - (key, val_) => key + (key, val_) => key ); } catch (error) { log.error( @@ -759,7 +770,11 @@ export async function* readDirRecursiveWithIgnore( for await (const e2 of e.c()) { if (e2.isDirectory) { const dirName = e2.path.split(SEP).pop(); - if (dirName == "node_modules" || dirName == ".claude" || dirName?.startsWith(".")) { + if ( + dirName == "node_modules" || + dirName == ".claude" || + dirName?.startsWith(".") + ) { continue; } } @@ -792,7 +807,7 @@ export async function elementsToMap( ignore: (path: string, isDirectory: boolean) => boolean, json: boolean, skips: Skips, - specificItems?: SpecificItemsConfig, + specificItems?: SpecificItemsConfig ): Promise<{ [key: string]: string }> { const map: { [key: string]: string } = {}; const processedBasePaths = new Set(); @@ -802,7 +817,11 @@ export async function elementsToMap( continue; } const path = entry.path; - if (!isFileResource(path) && !isRawAppFile(path) && !isWorkspaceDependencies(path)) { + if ( + !isFileResource(path) && + !isRawAppFile(path) && + !isWorkspaceDependencies(path) + ) { if (json && path.endsWith(".yaml")) continue; if (!json && path.endsWith(".json")) continue; @@ -835,7 +854,11 @@ export async function elementsToMap( if (isRawAppFile(path)) { const suffix = path.split(".raw_app" + SEP).pop(); - if (suffix?.startsWith("dist/") || suffix == "wmill.d.ts" || suffix == "package-lock.json") { + if ( + suffix?.startsWith("dist/") || + suffix == "wmill.d.ts" || + suffix == "package-lock.json" + ) { continue; } } @@ -867,7 +890,6 @@ export async function elementsToMap( continue; } - // Use getTypeStrFromPath for consistent type detection try { const fileType = getTypeStrFromPath(path); @@ -876,12 +898,15 @@ export async function elementsToMap( if (skips.skipFlows && fileType === "flow") continue; if (skips.skipApps && fileType === "app") continue; if (skips.skipFolders && fileType === "folder") continue; - if (skips.skipWorkspaceDependencies && fileType === "workspace_dependencies") continue; + if ( + skips.skipWorkspaceDependencies && + fileType === "workspace_dependencies" + ) + continue; } catch { // If getTypeStrFromPath can't determine the type, continue processing the file } - // Handle branch-specific files - skip files for other branches if (specificItems && isBranchSpecificFile(path)) { const currentBranch = getCurrentGitBranch(); @@ -945,7 +970,7 @@ export async function elementsToMap( // No specific items configuration, use regular path map[entry.path] = content; } - } + } return map; } @@ -1090,7 +1115,6 @@ async function compareDynFSElement( const remoteCodebase: Record = {}; for (const [k] of Object.entries(m2)) { - if (m1[k] === undefined) { if ( !ignoreMetadataDeletion || @@ -1239,7 +1263,7 @@ export const isWhitelisted = (p: string) => { p == "f" || p == "g" || p == "users" || - p == "groups" || + p == "groups" || p == "dependencies" ); }; @@ -1308,7 +1332,10 @@ export async function ignoreF(wmillconf: { if (wmillconf.includeKey && fileType === "encryption_key") { return false; // Don't ignore, always include } - if (!wmillconf.skipWorkspaceDependencies && fileType === "workspace_dependencies") { + if ( + !wmillconf.skipWorkspaceDependencies && + fileType === "workspace_dependencies" + ) { return false; // Don't ignore workspace dependencies (they are always included unless explicitly skipped) } } catch { @@ -1440,22 +1467,22 @@ export async function pull( // ignore } const zipFile = await downloadZip( - workspace, - opts.plainSecrets, - opts.skipVariables, - opts.skipResources, - opts.skipResourceTypes, - opts.skipSecrets, - opts.includeSchedules, - opts.includeTriggers, - opts.includeUsers, - opts.includeGroups, - opts.includeSettings, - opts.includeKey, - opts.skipWorkspaceDependencies, - opts.defaultTs - ); - + workspace, + opts.plainSecrets, + opts.skipVariables, + opts.skipResources, + opts.skipResourceTypes, + opts.skipSecrets, + opts.includeSchedules, + opts.includeTriggers, + opts.includeUsers, + opts.includeGroups, + opts.includeSettings, + opts.includeKey, + opts.skipWorkspaceDependencies, + opts.defaultTs + ); + const remote = ZipFSElement( zipFile!, !opts.json, @@ -1482,7 +1509,6 @@ export async function pull( `remote (${workspace.name}) -> local: ${changes.length} changes to apply` ); - // Handle JSON output for dry-run if (opts.dryRun && opts.jsonOutput) { const result = { @@ -1671,7 +1697,8 @@ export async function pull( await readLockfile(); // ensure wmill-lock.yaml exists const tracker: ChangeTracker = await buildTracker(changes); - const rawWorkspaceDependencies: Record = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies: Record = + await getRawWorkspaceDependencies(); for (const change of tracker.scripts) { await generateScriptMetadataInternal( @@ -1687,7 +1714,14 @@ export async function pull( } for (const change of tracker.flows) { log.info(`Updating lock for flow ${change}`); - await generateFlowLockInternal(change, false, workspace, opts, true, false); + await generateFlowLockInternal( + change, + false, + workspace, + opts, + true, + false + ); } if (tracker.apps.length > 0) { log.info( @@ -1700,7 +1734,14 @@ export async function pull( } for (const change of tracker.rawApps) { log.info(`Updating lock metadata for raw app ${change}`); - await generateAppLocksInternal(change, false, workspace, opts, true, true); + await generateAppLocksInternal( + change, + false, + workspace, + opts, + true, + true + ); } if (opts.jsonOutput) { const result = { @@ -1899,6 +1940,7 @@ export async function push( } catch { // ignore } + const remote = ZipFSElement( (await downloadZip( workspace, @@ -1935,13 +1977,13 @@ export async function push( specificItems ); - const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); const tracker: ChangeTracker = await buildTracker(changes); const staleScripts: string[] = []; const staleFlows: string[] = []; + for (const change of tracker.scripts) { const stale = await generateScriptMetadataInternal( change, @@ -1995,7 +2037,9 @@ export async function push( } const version = await fetchVersion(workspace.remote); - + if (version) { + updateGlobalVersions(version); + } log.info(colors.gray("Remote version: " + version)); log.info( @@ -2097,14 +2141,18 @@ export async function push( const alreadySynced: string[] = []; const isRawApp = isRawAppFile(changes[0].path); if (isRawApp) { - const deleteRawApp = changes.find(change => change.name === "deleted" && change.path.endsWith(".raw_app/raw_app.yaml")) + const deleteRawApp = changes.find( + (change) => + change.name === "deleted" && + change.path.endsWith(".raw_app/raw_app.yaml") + ); if (deleteRawApp) { changes = [deleteRawApp]; } else { changes.splice(1, changes.length - 1); } } - + for await (const change of changes) { let stateTarget = undefined; if (stateful) { @@ -2324,13 +2372,16 @@ export async function push( }); break; case "raw_app": - if (target.endsWith(".raw_app/raw_app.yaml") || target.endsWith(".raw_app/raw_app.json")) { + if ( + target.endsWith(".raw_app/raw_app.yaml") || + target.endsWith(".raw_app/raw_app.json") + ) { await wmill.deleteApp({ workspace: workspaceId, path: removeSuffix(target, ".raw_app/raw_app.json"), }); } - break; + break; case "schedule": await wmill.deleteSchedule({ workspace: workspaceId, @@ -2426,18 +2477,25 @@ export async function push( }); break; case "workspace_dependencies": - const relativePath = removePathPrefix(change.path, "dependencies"); - - const res = workspaceDependenciesPathToLanguageAndFilename(change.path); + const relativePath = removePathPrefix( + change.path, + "dependencies" + ); + + const res = workspaceDependenciesPathToLanguageAndFilename( + change.path + ); if (!res) { - throw new Error(`Unknown workspace dependencies file format: ${change.path}`); + throw new Error( + `Unknown workspace dependencies file format: ${change.path}` + ); } const { name, language } = res; await wmill.deleteWorkspaceDependencies({ workspace: workspaceId, language, - name + name, }); break; @@ -2537,7 +2595,10 @@ const command = new Command() .option("--skip-flows", "Skip syncing flows") .option("--skip-apps", "Skip syncing apps") .option("--skip-folders", "Skip syncing folders") - .option("--skip-workspace-dependencies", "Skip syncing workspace dependencies") + .option( + "--skip-workspace-dependencies", + "Skip syncing workspace dependencies" + ) // .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic") .option("--include-schedules", "Include syncing schedules") .option("--include-triggers", "Include syncing triggers") @@ -2586,7 +2647,10 @@ const command = new Command() .option("--skip-flows", "Skip syncing flows") .option("--skip-apps", "Skip syncing apps") .option("--skip-folders", "Skip syncing folders") - .option("--skip-workspace-dependencies", "Skip syncing workspace dependencies") + .option( + "--skip-workspace-dependencies", + "Skip syncing workspace dependencies" + ) // .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic") .option("--include-schedules", "Include syncing schedules") .option("--include-triggers", "Include syncing triggers") From 69c550bca6dffb64bd8150648a6d358a25a018fd Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:48:14 +0100 Subject: [PATCH 07/10] fix(workspace-dependencies): implement better caching (#7273) * fix(workspace-dependencies): implement better caching Signed-off-by: pyranota * Update SQLx metadata * nits Signed-off-by: pyranota * spawn non blocking Signed-off-by: pyranota * nit Signed-off-by: pyranota * fix ci Signed-off-by: pyranota * fix issue Signed-off-by: pyranota --------- Signed-off-by: pyranota Co-authored-by: windmill-internal-app[bot] --- ...9649d0f5db125776fbdffea993a53a09708de.json | 55 ++++++ ...80766003d79884d33f94a79e61f0259807dbc.json | 128 ------------- backend/tests/workspace_dependencies.rs | 4 +- .../src/workspace_dependencies.rs | 73 ++++---- backend/windmill-common/src/cache.rs | 83 ++++++++- .../src/workspace_dependencies.rs | 174 ++++++++++++------ .../src/workspace_dependencies.rs | 124 ++++++++----- 7 files changed, 366 insertions(+), 275 deletions(-) create mode 100644 backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json delete mode 100644 backend/.sqlx/query-f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc.json diff --git a/backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json b/backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json new file mode 100644 index 0000000000..b014854b40 --- /dev/null +++ b/backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + ] + }, + "nullable": [ + false + ] + }, + "hash": "5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de" +} diff --git a/backend/.sqlx/query-f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc.json b/backend/.sqlx/query-f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc.json deleted file mode 100644 index ab4421f9b4..0000000000 --- a/backend/.sqlx/query-f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT id, content, language AS \"language: ScriptLang\", name, description, archived, workspace_id, created_at\n FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3\n LIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "content", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "language: ScriptLang", - "type_info": { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "description", - "type_info": "Text" - }, - { - "ordinal": 5, - "name": "archived", - "type_info": "Bool" - }, - { - "ordinal": 6, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - } - ] - }, - "nullable": [ - false, - false, - false, - true, - false, - false, - false, - false - ] - }, - "hash": "f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc" -} diff --git a/backend/tests/workspace_dependencies.rs b/backend/tests/workspace_dependencies.rs index 6875520125..b41eb7c550 100644 --- a/backend/tests/workspace_dependencies.rs +++ b/backend/tests/workspace_dependencies.rs @@ -64,7 +64,7 @@ mod workspace_dependencies { name: Some("test".to_owned()), description: None, } - .create("", "", "", db) + .create(("".to_owned(), "".to_owned(), "".to_owned()), db.clone()) .await .unwrap(); @@ -151,7 +151,7 @@ mod workspace_dependencies { name: None, // No name = default workspace dependencies description: None, } - .create("", "", "", db) + .create(("".to_owned(), "".to_owned(), "".to_owned()), db.clone()) .await .unwrap(); diff --git a/backend/windmill-api/src/workspace_dependencies.rs b/backend/windmill-api/src/workspace_dependencies.rs index bb511518d2..2330de200a 100644 --- a/backend/windmill-api/src/workspace_dependencies.rs +++ b/backend/windmill-api/src/workspace_dependencies.rs @@ -13,9 +13,8 @@ use windmill_common::{ workspace_dependencies::WorkspaceDependencies, DB, }; -use windmill_worker::{ - scoped_dependency_map, trigger_dependents_to_recompute_dependencies, - workspace_dependencies::NewWorkspaceDependencies, +use windmill_worker::workspace_dependencies::{ + trigger_dependents_to_recompute_dependencies_in_the_background, NewWorkspaceDependencies, }; use crate::db::ApiAuthed; @@ -43,10 +42,12 @@ async fn create( format!( "{}", nwd.create( - &authed.email, - &authed.username, - &username_to_permissioned_as(&authed.username), - &db + ( + authed.email, + username_to_permissioned_as(&authed.username), + authed.username, + ), + db ) .await? ), @@ -93,23 +94,20 @@ async fn archive( let db = &db; WorkspaceDependencies::archive(params.name.clone(), language, &w_id, db).await?; - trigger_dependents_to_recompute_dependencies( - &w_id, - scoped_dependency_map::ScopedDependencyMap::get_dependents( - WorkspaceDependencies::to_path(¶ms.name, language)?.as_str(), - &w_id, - db, - ) - .await?, - None, - None, - &authed.email, - &authed.username, - &username_to_permissioned_as(&authed.username), - db, - vec![], + trigger_dependents_to_recompute_dependencies_in_the_background( + params.name.is_none(), + w_id, + language, + ( + authed.email, + username_to_permissioned_as(&authed.username), + authed.username, + ), + WorkspaceDependencies::to_path(¶ms.name, language)?, + db.clone(), ) - .await + .await; + Ok(()) } #[axum::debug_handler] @@ -125,21 +123,18 @@ async fn delete( let db = &db; WorkspaceDependencies::delete(params.name.clone(), language, &w_id, db).await?; - trigger_dependents_to_recompute_dependencies( - &w_id, - scoped_dependency_map::ScopedDependencyMap::get_dependents( - WorkspaceDependencies::to_path(¶ms.name, language)?.as_str(), - &w_id, - db, - ) - .await?, - None, - None, - &authed.email, - &authed.username, - &username_to_permissioned_as(&authed.username), - db, - vec![], + trigger_dependents_to_recompute_dependencies_in_the_background( + params.name.is_none(), + w_id, + language, + ( + authed.email, + username_to_permissioned_as(&authed.username), + authed.username, + ), + WorkspaceDependencies::to_path(¶ms.name, language)?, + db.clone(), ) - .await + .await; + Ok(()) } diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 8fef4b5e3a..3cfc98dc98 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -892,6 +892,86 @@ pub mod job { } } +pub mod workspace_dependencies { + use std::{ + future::Future, + time::{Duration, Instant}, + }; + + use crate::{error, scripts::ScriptLang, workspace_dependencies::WorkspaceDependencies, DB}; + + make_static! { + /// Workspace Dependencies by id and workspace cache. + static ref WORKSPACE_DEPENDENCIES: { (i64, String) => WorkspaceDependencies } in "workspace_dependencies" <= 1000; + } + lazy_static::lazy_static! { + /// Cache for checking if default/unnamed workspace dependencies exist for a workspace and language. + /// Cache key: (workspace_id, language) + /// Cache value: (exists: bool, cached_at timestamp) + static ref DEFAULT_WD_EXISTS_CACHE: quick_cache::sync::Cache<(String, ScriptLang), (bool, Instant)> = quick_cache::sync::Cache::new(500); + } + /// Cache timeout for existence checks (10 seconds) + pub const EXISTS_CACHE_TIMEOUT: Duration = Duration::from_secs(10); + + pub fn fetch_workspace_dependencies<'c>( + id: i64, + workspace_id: String, + db: &'c DB, + ) -> impl Future> + 'c { + tracing::debug!(workspace_id = %workspace_id, id, "fetching workspace dependencies"); + WORKSPACE_DEPENDENCIES.get_or_insert_async( + (id, workspace_id.clone()), + WorkspaceDependencies::get(id, workspace_id, db), + ) + } + + pub fn get_cached_is_unnamed_workspace_dependencies_exists<'c>( + language: ScriptLang, + workspace_id: String, + ) -> Option { + let exists_key = (workspace_id.to_string(), language); + if let Some((exists, cached_at)) = DEFAULT_WD_EXISTS_CACHE.get(&exists_key) { + if cached_at.elapsed() < EXISTS_CACHE_TIMEOUT { + tracing::debug!( + workspace_id = %workspace_id, + ?language, + exists, + "cache hit for unnamed workspace dependencies existence" + ); + return Some(exists); + } else { + tracing::debug!( + workspace_id = %workspace_id, + ?language, + "cache expired for unnamed workspace dependencies existence" + ); + DEFAULT_WD_EXISTS_CACHE.remove(&exists_key); + } + } else { + tracing::debug!( + workspace_id = %workspace_id, + ?language, + "cache miss for unnamed workspace dependencies existence" + ); + } + None + } + pub fn set_cached_is_unnamed_workspace_dependencies_exists<'c>( + language: ScriptLang, + workspace_id: String, + exists: bool, + ) { + tracing::debug!( + workspace_id = %workspace_id, + ?language, + exists, + "setting cache for unnamed workspace dependencies existence" + ); + let exists_key = (workspace_id.to_string(), language); + DEFAULT_WD_EXISTS_CACHE.insert(exists_key, (exists, Instant::now())); + } +} + const _: () = { impl Import for RawFlow { fn import(src: &impl Storage) -> error::Result { @@ -1075,7 +1155,8 @@ const _: () = { (ScriptHash, |x| format!("{:016x}", x.0)), ((u8, ScriptHash), |x| format!("{:02x}-{:016x}", x.0, x.1.0)), (FlowNodeId, |x| format!("{:016x}", x.0)), - (AppScriptId, |x| format!("{:016x}", x.0)) + (AppScriptId, |x| format!("{:016x}", x.0)), + ((i64, String), |x| format!("{}-{}", x.1, x.0)) } #[cfg(feature = "scoped_cache")] diff --git a/backend/windmill-common/src/workspace_dependencies.rs b/backend/windmill-common/src/workspace_dependencies.rs index c264f249f7..8060a6ac98 100644 --- a/backend/windmill-common/src/workspace_dependencies.rs +++ b/backend/windmill-common/src/workspace_dependencies.rs @@ -2,9 +2,17 @@ use itertools::Itertools; use regex::Regex; use serde::{Deserialize, Serialize}; use sqlx::PgExecutor; -use std::time::{Duration, Instant}; -use crate::{error, scripts::ScriptLang, utils::calculate_hash, worker::Connection}; +use crate::{ + cache::workspace_dependencies::{ + fetch_workspace_dependencies, get_cached_is_unnamed_workspace_dependencies_exists, + set_cached_is_unnamed_workspace_dependencies_exists, + }, + error, + scripts::ScriptLang, + utils::calculate_hash, + worker::Connection, +}; use phf::phf_set; pub static BLACKLIST: phf::Set<&'static str> = phf_set! { @@ -15,16 +23,8 @@ pub static BLACKLIST: phf::Set<&'static str> = phf_set! { lazy_static::lazy_static! { static ref WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: bool = std::env::var("WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES").is_ok(); - - /// Simple in-memory cache for workspace dependencies get_latest with 10-second timeout. - /// Cache key: (workspace_id, language, name) - /// Cache value: (Option, cached_at timestamp) - static ref WORKSPACE_DEPENDENCIES_CACHE: quick_cache::sync::Cache<(String, ScriptLang, Option), (Option, Instant)> = quick_cache::sync::Cache::new(1000); } -/// Cache timeout for workspace dependencies -const CACHE_TIMEOUT: Duration = Duration::from_secs(10); - /// Minimum Windmill version required for workspace dependencies feature pub const MIN_VERSION_WORKSPACE_DEPENDENCIES: &str = "1.587.0"; @@ -174,6 +174,42 @@ impl WorkspaceDependencies { .map_err(error::Error::from) } + async fn get_latest_id<'c>( + name: Option, + language: ScriptLang, + workspace_id: &str, + e: impl PgExecutor<'c>, + ) -> error::Result> { + tracing::debug!( + workspace_id = %workspace_id, + ?language, + ?name, + "fetching latest workspace dependencies id" + ); + let result = sqlx::query_scalar!( + r#" + SELECT id FROM workspace_dependencies + WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3 + LIMIT 1 + "#, + name, + workspace_id, + language as ScriptLang + ) + .fetch_optional(e) + .await + .map_err(error::Error::from)?; + + tracing::debug!( + workspace_id = %workspace_id, + ?language, + ?name, + ?result, + "fetched latest workspace dependencies id" + ); + Ok(result) + } + /// Gets the latest version of workspace dependencies by name and language. pub async fn get_latest( name: Option, @@ -185,66 +221,84 @@ impl WorkspaceDependencies { return Ok(None); } - let cache_key = (workspace_id.to_string(), language, name.clone()); - - // Check if cached value is still valid - if let Some((cached_value, cached_at)) = WORKSPACE_DEPENDENCIES_CACHE.get(&cache_key) { - if cached_at.elapsed() < CACHE_TIMEOUT { - return Ok(cached_value); - } - // Expired, remove it - WORKSPACE_DEPENDENCIES_CACHE.remove(&cache_key); + if name.is_none() + && get_cached_is_unnamed_workspace_dependencies_exists( + language, + workspace_id.to_owned(), + ) + .map(|exists| exists == false) + .unwrap_or_default() + { + tracing::debug!( + workspace_id = %workspace_id, + ?language, + "skipping unnamed workspace dependencies fetch - cached as non-existent" + ); + return Ok(None); } - // Fetch and cache - let fetch = Box::pin(async { - match &conn { - Connection::Sql(db) => sqlx::query_as!( - Self, - r#" - SELECT id, content, language AS "language: ScriptLang", name, description, archived, workspace_id, created_at - FROM workspace_dependencies - WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3 - LIMIT 1 - "#, - name, - workspace_id, - language as ScriptLang - ) - .fetch_optional(db) - .await - .map_err(error::Error::from), + // Fetch from database or HTTP + let wd = match &conn { + Connection::Sql(db) => { + let Some(id) = + Self::get_latest_id(name.clone(), language, workspace_id, db).await? + else { + if name.is_none() { + set_cached_is_unnamed_workspace_dependencies_exists( + language, + workspace_id.to_owned(), + false, + ); + } - Connection::Http(http_client) => http_client - .get::>(&format!( - "/api/w/{workspace_id}/agent_workers/workspace_dependencies/get_latest/{}{}", - language.as_str(), - if let Some(ref name_val) = name { - format!("?name={name_val}") - } else { - "".to_owned() - } - )) - .await - .map_err(error::Error::from), + tracing::debug!( + workspace_id = %workspace_id, + ?language, + ?name, + "no latest workspace dependencies found" + ); + return Ok(None); + }; + tracing::debug!( + workspace_id = %workspace_id, + ?language, + ?name, + id, + "fetching workspace dependencies by id from cache or db" + ); + Some(fetch_workspace_dependencies(id, workspace_id.to_owned(), db).await?) } - }); - let (workspace_dependencies_o, ..) = WORKSPACE_DEPENDENCIES_CACHE - .get_or_insert_async(&cache_key, async { - Ok::<_, error::Error>((fetch.await?, Instant::now())) - }) - .await?; + Connection::Http(http_client) => http_client + .get::>(&format!( + "/api/w/{workspace_id}/agent_workers/workspace_dependencies/get_latest/{}{}", + language.as_str(), + if let Some(ref name_val) = name { + format!("?name={name_val}") + } else { + "".to_owned() + } + )) + .await + .map_err(error::Error::from)?, + }; - Ok(workspace_dependencies_o) + if name.is_none() { + set_cached_is_unnamed_workspace_dependencies_exists( + language, + workspace_id.to_owned(), + wd.is_some(), + ); + } + Ok(wd) } /// Gets workspace dependencies by their unique ID. pub async fn get<'c>( id: i64, - workspace_id: &str, + workspace_id: String, e: impl PgExecutor<'c>, - ) -> error::Result> { + ) -> error::Result { sqlx::query_as!( Self, r#" @@ -254,9 +308,9 @@ impl WorkspaceDependencies { LIMIT 1 "#, id, - workspace_id + &workspace_id ) - .fetch_optional(e) + .fetch_one(e) .await .map_err(error::Error::from) } diff --git a/backend/windmill-worker/src/workspace_dependencies.rs b/backend/windmill-worker/src/workspace_dependencies.rs index dd060122a7..281b08a32c 100644 --- a/backend/windmill-worker/src/workspace_dependencies.rs +++ b/backend/windmill-worker/src/workspace_dependencies.rs @@ -1,5 +1,8 @@ use serde::{Deserialize, Serialize}; -use windmill_common::{error, scripts::ScriptLang, workspace_dependencies::WorkspaceDependencies}; +use windmill_common::{ + cache::workspace_dependencies::EXISTS_CACHE_TIMEOUT, error, scripts::ScriptLang, + workspace_dependencies::WorkspaceDependencies, +}; use crate::{ scoped_dependency_map::ScopedDependencyMap, trigger_dependents_to_recompute_dependencies, @@ -25,10 +28,8 @@ impl NewWorkspaceDependencies { /// and rebuilds the dependency map if this is the first unnamed dependency for the workspace. pub async fn create<'c>( self, - email: &str, - created_by: &str, - permissioned_as: &str, - db: &sqlx::Pool, + metadata: (String, String, String), + db: sqlx::Pool, ) -> error::Result { // Check if all workers support workspace dependencies feature windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies() @@ -43,7 +44,7 @@ impl NewWorkspaceDependencies { let setting_name = format!("workspace_dependencies_map_rebuilt:{}", self.workspace_id); let already_rebuilt = windmill_common::global_settings::load_value_from_global_settings( - db, + &db, &setting_name, ) .await? @@ -54,11 +55,11 @@ impl NewWorkspaceDependencies { workspace_id = %self.workspace_id, "Rebuilding workspace dependencies map for first unnamed workspace dependencies" ); - ScopedDependencyMap::rebuild_map_unchecked(&self.workspace_id, db).await?; + ScopedDependencyMap::rebuild_map_unchecked(&self.workspace_id, &db).await?; // Mark as rebuilt by creating the setting windmill_common::global_settings::set_value_in_global_settings( - db, + &db, &setting_name, serde_json::json!({}), ) @@ -111,50 +112,83 @@ impl NewWorkspaceDependencies { .await?; tx.commit().await?; - // Make sure trigger dependents will have latest view. - // NOTE: Uncomment for tests - // #[cfg(test)] - // assert_eq!( - // sqlx::query_scalar!( - // " - // SELECT id FROM workspace_dependencies - // WHERE archived = false - // AND name IS NOT DISTINCT FROM $1 - // AND workspace_id = $2 - // AND language = $3 - // ", - // self.name, - // self.workspace_id, - // self.language as ScriptLang, - // ) - // .fetch_one(db) // Use db - // .await?, - // new_id - // ); - - // It's ok to fail, it will return an error and user will get notified that they should redeploy workspace dependencies - trigger_dependents_to_recompute_dependencies( - &self.workspace_id, - crate::scoped_dependency_map::ScopedDependencyMap::get_dependents( - path.as_str(), - &self.workspace_id, - db, - ) - .await?, - None, - None, - email, - created_by, - permissioned_as, + trigger_dependents_to_recompute_dependencies_in_the_background( + prev_description.is_none() && self.name.is_none(), + self.workspace_id, + self.language, + metadata, + path, db, - vec![], ) - .await?; + .await; Ok(new_id) } } +pub async fn trigger_dependents_to_recompute_dependencies_in_the_background( + wait_for_cache_timeout: bool, + workspace_id: String, + language: ScriptLang, + (email, permissioned_as, created_by): (String, String, String), + path: String, + db: sqlx::Pool, +) { + tokio::spawn(async move { + if wait_for_cache_timeout { + tracing::debug!( + workspace_id = %workspace_id, + language = ?language, + "waiting for cache timeout after creating first unnamed workspace dependencies" + ); + // Wait for cache timeout. + // For context, workers have cache on whether the unnamed workspace dependencies exists or not. + // when we trigger dependents to recompoute dependencies we want to make sure all workers are having cache timed out. + // otherwise it would result into bug, when workers skip fetch of workspace dependencies because they think they don't exist. + tokio::time::sleep(EXISTS_CACHE_TIMEOUT).await; + } + + // It's ok to fail, it will return an error and user will get notified that they should redeploy workspace dependencies + if let Err(e) = trigger_dependents_to_recompute_dependencies( + &workspace_id, + match crate::scoped_dependency_map::ScopedDependencyMap::get_dependents( + path.as_str(), + &workspace_id, + &db, + ) + .await + { + Ok(importers) => importers, + Err(e) => { + tracing::error!( + workspace_id = %workspace_id, + path = %path, + error = %e, + "CRITICAL: failed to get dependents for workspace dependencies - dependent runnables are not being redeployed. Please contact the Windmill team" + ); + return; + } + }, + None, + None, + email.as_str(), + created_by.as_str(), + permissioned_as.as_str(), + &db, + vec![], + ) + .await + { + tracing::error!( + workspace_id = %workspace_id, + path = %path, + error = %e, + "CRITICAL: failed to trigger dependents to recompute dependencies - dependent runnables are not being redeployed. Please contact the Windmill team" + ); + } + }); +} + // Type aliases for backward compatibility pub type RawRequirements = WorkspaceDependencies; pub type NewRawRequirements = NewWorkspaceDependencies; From a23d4f015aa32e00fd96b8f49bccd37efa1200b2 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 2 Dec 2025 13:53:35 +0100 Subject: [PATCH 08/10] feat: add S3 support to download button and PDF preview components (#7271) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add S3 support to download button and PDF preview components Add S3 object and s3:// URL support to AppDownload and AppPdf components, following the same pattern used in AppImage component. Both components now: - Handle partial S3 objects with storage and presigned URL support - Handle s3:// URL format - Construct proper API endpoints for S3 file downloads Fixes #7240 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Co-authored-by: windmill-internal-app[bot] * handle policy + fix s3 picker --------- Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: Claude Co-authored-by: windmill-internal-app[bot] --- frontend/src/lib/components/EditorBar.svelte | 4 +- .../src/lib/components/S3FilePicker.svelte | 17 +++--- .../src/lib/components/S3ObjectPicker.svelte | 2 +- .../components/display/AppDownload.svelte | 48 ++++++++++++++- .../apps/components/display/AppImage.svelte | 61 +++++++------------ .../apps/components/display/AppPdf.svelte | 45 +++++++++++++- .../lib/components/apps/editor/appPolicy.ts | 11 +++- .../lib/components/apps/editor/appUtilsS3.ts | 57 ++++++++++++++++- .../apps/editor/component/components.ts | 9 ++- .../settingsPanel/InputsSpecEditor.svelte | 8 ++- .../inputEditor/StaticInputEditor.svelte | 18 +++--- .../common/fileUpload/S3ArgInput.svelte | 10 +-- .../triggers/http/RouteEditorInner.svelte | 3 +- 13 files changed, 215 insertions(+), 78 deletions(-) diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 1b3d6ee8a1..315bfeba75 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -757,8 +757,8 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS { - let s = `'${formatS3Object(s3obj.detail)}'` + onSelectAndClose={(s3obj) => { + let s = `'${formatS3Object(s3obj)}'` if (lang === 'duckdb') { editor?.insertAtCursor(`SELECT * FROM ${s}`) } else if (lang === 'python3') { diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index cc061dc18c..16eddb9f1a 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -2,7 +2,7 @@ import { emptyString, type S3Object } from '$lib/utils' import { Button, Drawer } from './common' import DrawerContent from './common/drawer/DrawerContent.svelte' - import { createEventDispatcher, tick, untrack } from 'svelte' + import { tick, untrack } from 'svelte' import S3FilePickerInner from './S3FilePickerInner.svelte' import Select from './select/Select.svelte' import { FileUp } from 'lucide-svelte' @@ -10,11 +10,6 @@ import { SettingService } from '$lib/gen' import { workspaceStore } from '$lib/stores' - let dispatch = createEventDispatcher<{ - close: { s3: string; storage: string | undefined } | undefined - selectAndClose: { s3: string; storage: string | undefined } - }>() - interface Props { fromWorkspaceSettings?: boolean readOnlyMode: boolean @@ -22,6 +17,8 @@ selectedFileKey?: { s3: string; storage?: string } | undefined folderOnly?: boolean regexFilter?: RegExp | undefined + onClose?: () => void + onSelectAndClose?: (selected: { s3: string; storage: string | undefined }) => void } let { @@ -30,7 +27,9 @@ initialFileKey = $bindable(undefined), selectedFileKey = $bindable(undefined), folderOnly = false, - regexFilter = undefined + regexFilter = undefined, + onClose, + onSelectAndClose }: Props = $props() let drawer: Drawer | undefined = $state() @@ -76,7 +75,7 @@ { - dispatch('close') + onClose?.() s3FilePickerInner?.close?.() }} size="1200px" @@ -93,7 +92,7 @@ { - dispatch('selectAndClose', e.detail) + onSelectAndClose?.(e.detail) drawer?.closeDrawer?.() }} {fromWorkspaceSettings} diff --git a/frontend/src/lib/components/S3ObjectPicker.svelte b/frontend/src/lib/components/S3ObjectPicker.svelte index 6a9d1c9a86..c70983ee83 100644 --- a/frontend/src/lib/components/S3ObjectPicker.svelte +++ b/frontend/src/lib/components/S3ObjectPicker.svelte @@ -39,7 +39,7 @@ { + onClose={() => { rawValue = JSON.stringify(value, null, 2) editor?.setCode(rawValue) }} diff --git a/frontend/src/lib/components/apps/components/display/AppDownload.svelte b/frontend/src/lib/components/apps/components/display/AppDownload.svelte index dbc732bd06..0f38afe1db 100644 --- a/frontend/src/lib/components/apps/components/display/AppDownload.svelte +++ b/frontend/src/lib/components/apps/components/display/AppDownload.svelte @@ -12,6 +12,8 @@ import ComponentErrorHandler from '../helpers/ComponentErrorHandler.svelte' import ResolveStyle from '../helpers/ResolveStyle.svelte' import AlignWrapper from '../helpers/AlignWrapper.svelte' + import { userStore } from '$lib/stores' + import { isPartialS3Object, getS3File } from '../../editor/appUtilsS3' interface Props { id: string @@ -37,7 +39,8 @@ initConfig(components['downloadcomponent'].initialData.configuration, configuration) ) - const { app, worldStore } = getContext('AppViewerContext') + const { app, worldStore, appPath, workspace, isEditor } = + getContext('AppViewerContext') //used so that we can count number of outputs setup for first refresh initOutput($worldStore, id, {}) @@ -45,6 +48,10 @@ let beforeIconComponent: any = $state() let afterIconComponent: any = $state() + let downloadUrl: string | undefined = $state(undefined) + + let token = getContext<{ token?: string }>('AuthToken') + async function handleBeforeIcon() { if (resolvedConfig.beforeIcon) { beforeIconComponent = await loadIcon( @@ -69,6 +76,36 @@ } } + async function loadSource() { + if (isPartialS3Object(resolvedConfig.source)) { + downloadUrl = await getS3File({ + source: resolvedConfig.source.s3, + storage: resolvedConfig.source.storage, + presigned: resolvedConfig.source.presigned, + appPath: $appPath, + username: $userStore?.username, + workspace, + token: token?.token, + isEditor, + configuration + }) + } else if (resolvedConfig.source && typeof resolvedConfig.source !== 'string') { + throw new Error('Invalid source object' + typeof resolvedConfig.source) + } else if (resolvedConfig.source?.startsWith('s3://')) { + downloadUrl = await getS3File({ + source: resolvedConfig.source?.replace('s3://', ''), + appPath: $appPath, + username: $userStore?.username, + workspace, + token: token?.token, + isEditor, + configuration + }) + } else { + downloadUrl = transformBareBase64IfNecessary(resolvedConfig.source) + } + } + let css = $state(initCss($app.css?.downloadcomponent, customCss)) $effect(() => { resolvedConfig.beforeIcon && beforeIconComponent && untrack(() => handleBeforeIcon()) @@ -76,6 +113,9 @@ $effect(() => { resolvedConfig.afterIcon && afterIconComponent && untrack(() => handleAfterIcon()) }) + $effect(() => { + resolvedConfig && loadSource() + }) @@ -102,7 +142,9 @@ {#if render}