diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 282cba946a..54702f7f32 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,4 @@ -* @rubenfiszel @HugoCasa @alpetric +* @rubenfiszel @hugocasa @alpetric -/community/ @rubenfiszel @HugoCasa @alpetric -/frontend/ @rubenfiszel @HugoCasa @alpetric +/community/ @rubenfiszel @hugocasa @alpetric +/frontend/ @rubenfiszel @hugocasa @alpetric diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 7fda0e025f..dd8452e2cf 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -42,7 +42,7 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER RUN /usr/local/bin/python3 -m pip install pip-tools # Bun -COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun ARG TARGETPLATFORM diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 738b2a9224..446606503d 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -33,7 +33,7 @@ jobs: - name: Setup Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.85.0 + toolchain: 1.88.0 override: true - name: Substitute EE code diff --git a/.github/workflows/git-commands.yaml b/.github/workflows/git-commands.yaml index 4dbdf36ed1..9dcae33bb1 100644 --- a/.github/workflows/git-commands.yaml +++ b/.github/workflows/git-commands.yaml @@ -151,17 +151,17 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - echo "${{ steps.get-commit-hash.outputs.commit_hash }}" > backend/ee-repo-ref.txt - echo "Updated backend/ee-repo-ref.txt with commit hash: ${{ steps.get-commit-hash.outputs.commit_hash }}" - # commit and push the changes - PR_NUMBER=${{ github.event.issue.number }} - BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) echo "Checking out PR branch: $BRANCH_NAME" git checkout $BRANCH_NAME git config --local user.email "action@github.com" git config --local user.name "GitHub Action" git config pull.rebase true git pull origin $BRANCH_NAME + echo "${{ steps.get-commit-hash.outputs.commit_hash }}" > backend/ee-repo-ref.txt + echo "Updated backend/ee-repo-ref.txt with commit hash: ${{ steps.get-commit-hash.outputs.commit_hash }}" + # commit and push the changes + PR_NUMBER=${{ github.event.issue.number }} + BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) git add backend/ee-repo-ref.txt git commit -m "Update ee-repo-ref.txt" || echo "No changes to commit" git push origin $BRANCH_NAME diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index e9a8c77e87..809e610d58 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -35,7 +35,7 @@ jobs: - name: Setup Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.85.0 + toolchain: 1.88.0 override: true - name: Substitute EE code diff --git a/CHANGELOG.md b/CHANGELOG.md index 29f5b49b55..9f3b540dce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## [1.508.0](https://github.com/windmill-labs/windmill/compare/v1.507.2...v1.508.0) (2025-07-16) + + +### Features + +* **aichat:** add logs api endpoint as tool ([#6197](https://github.com/windmill-labs/windmill/issues/6197)) ([827e06b](https://github.com/windmill-labs/windmill/commit/827e06b4b3dd7b2b970b3a0a37e3dfbe98d1fafd)) +* better explanation for dev key renewal ([#6209](https://github.com/windmill-labs/windmill/issues/6209)) ([1d62dd0](https://github.com/windmill-labs/windmill/commit/1d62dd0fc323e484ab1d08e740379c8243b52df8)) + + +### Bug Fixes + +* **frontend:** add error handler on trigger delete ([#6208](https://github.com/windmill-labs/windmill/issues/6208)) ([c076d13](https://github.com/windmill-labs/windmill/commit/c076d1332e0871b16b5881e2cf3ef165bc9f5dd5)) +* **frontend:** enable delete triggers on detail page ([#6206](https://github.com/windmill-labs/windmill/issues/6206)) ([290daec](https://github.com/windmill-labs/windmill/commit/290daec0fa53f77f6ad02e746d82aa30997d8cb0)) +* hide s3 catalog picker in anonymous apps ([#6204](https://github.com/windmill-labs/windmill/issues/6204)) ([b68193f](https://github.com/windmill-labs/windmill/commit/b68193f80468aeeadff5af0826cca2f0cc9f05e8)) +* support enum for array list as multiselect ([07b785d](https://github.com/windmill-labs/windmill/commit/07b785dcc383ae87eaec65e765177e45fff0af20)) + +## [1.507.2](https://github.com/windmill-labs/windmill/compare/v1.507.1...v1.507.2) (2025-07-16) + + +### Bug Fixes + +* cancel autocomplete on escape + autocomplete qol ([#6201](https://github.com/windmill-labs/windmill/issues/6201)) ([038c179](https://github.com/windmill-labs/windmill/commit/038c179a432d4f1997a761392e27b16dea829e06)) +* **cli:** fix cli --skip-resources --skip-variables ([6c477c1](https://github.com/windmill-labs/windmill/commit/6c477c109bf7e803ec07d4485dec426b770c7f76)) + ## [1.507.1](https://github.com/windmill-labs/windmill/compare/v1.507.0...v1.507.1) (2025-07-16) diff --git a/Dockerfile b/Dockerfile index 934ef10148..3815744d1e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -194,7 +194,7 @@ COPY --from=builder /windmill/target/release/windmill ${APP}/windmill COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno -COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/backend/.sqlx/query-00ec0814dc270ee08d8cd818815681862711af1e6b046c4ff479622c153b69ed.json b/backend/.sqlx/query-00ec0814dc270ee08d8cd818815681862711af1e6b046c4ff479622c153b69ed.json new file mode 100644 index 0000000000..6b5950359e --- /dev/null +++ b/backend/.sqlx/query-00ec0814dc270ee08d8cd818815681862711af1e6b046c4ff479622c153b69ed.json @@ -0,0 +1,68 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE\n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n CASE\n -- flow step:\n WHEN flow_step_id IS NOT NULL THEN NULL\n -- completed:\n WHEN c.id IS NOT NULL THEN COALESCE(\n c.workflow_as_code_status || c.flow_status,\n c.workflow_as_code_status,\n c.flow_status\n )\n -- not completed:\n ELSE COALESCE(\n f.workflow_as_code_status || f.flow_status,\n f.workflow_as_code_status,\n f.flow_status\n )\n END AS \"flow_status: sqlx::types::Json>\",\n job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "completed", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "running", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "logs", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "mem_peak", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 5, + "name": "log_offset", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "progress", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Text", + "Uuid", + "Bool", + "Bool" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + false, + null + ] + }, + "hash": "00ec0814dc270ee08d8cd818815681862711af1e6b046c4ff479622c153b69ed" +} diff --git a/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json b/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json index 1842f8ab06..66a2152ae9 100644 --- a/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json +++ b/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861.json b/backend/.sqlx/query-140f4ad799fca6c25975a0aca7c9051f0760e22eccdd291c83ed86599ce571cb.json similarity index 67% rename from backend/.sqlx/query-2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861.json rename to backend/.sqlx/query-140f4ad799fca6c25975a0aca7c9051f0760e22eccdd291c83ed86599ce571cb.json index 2ebfb49b8e..754eb02825 100644 --- a/backend/.sqlx/query-2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861.json +++ b/backend/.sqlx/query-140f4ad799fca6c25975a0aca7c9051f0760e22eccdd291c83ed86599ce571cb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_status (id, workflow_as_code_status)\n VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3))\n ON CONFLICT (id) DO UPDATE SET\n workflow_as_code_status = JSONB_SET(\n COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB), \n array[$2],\n $3\n )", + "query": "INSERT INTO v2_job_status (id, workflow_as_code_status)\n VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3))\n ON CONFLICT (id) DO UPDATE SET\n workflow_as_code_status = JSONB_SET(\n COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB),\n array[$2],\n $3\n )", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861" + "hash": "140f4ad799fca6c25975a0aca7c9051f0760e22eccdd291c83ed86599ce571cb" } diff --git a/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json b/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json index 8217ec4576..197298f788 100644 --- a/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json +++ b/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json b/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json new file mode 100644 index 0000000000..e8b39caef7 --- /dev/null +++ b/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json @@ -0,0 +1,91 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Varchar", + "Int8Array", + "Text", + "Text", + "Text", + "Varchar", + "Text", + "Bool", + "Jsonb", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb" + ] + } + } + }, + { + "Custom": { + "name": "script_kind", + "kind": { + "Enum": [ + "script", + "trigger", + "failure", + "command", + "approval", + "preprocessor" + ] + } + } + }, + "Varchar", + "Bool", + "VarcharArray", + "Int4", + "Int4", + "Int4", + "Bool", + "Bool", + "Int2", + "Bool", + "Bool", + "Int4", + "Varchar", + "Bool", + "Bool", + "Varchar", + "Bool", + "Text", + "Bool", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e" +} diff --git a/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json b/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json index c4c33fe959..d64d8f404b 100644 --- a/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json +++ b/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33.json b/backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json similarity index 76% rename from backend/.sqlx/query-47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33.json rename to backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json index 9e1d80c2f6..08bb98cc06 100644 --- a/backend/.sqlx/query-47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33.json +++ b/backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\"\n FROM v2_as_completed_job \n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "query": "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\"\n FROM v2_as_completed_job\n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", "describe": { "columns": [ { @@ -26,5 +26,5 @@ true ] }, - "hash": "47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33" + "hash": "2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9" } diff --git a/backend/.sqlx/query-95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a.json b/backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json similarity index 72% rename from backend/.sqlx/query-95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a.json rename to backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json index d451727f46..ee5e15118d 100644 --- a/backend/.sqlx/query-95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a.json +++ b/backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_as_queue \n LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id \n WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", + "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_as_queue\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id\n WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", "describe": { "columns": [ { @@ -38,5 +38,5 @@ true ] }, - "hash": "95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a" + "hash": "2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f" } diff --git a/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json b/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json index e3b0e9a8da..31700319e2 100644 --- a/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json +++ b/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json index 1fa370e682..75b8108281 100644 --- a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json +++ b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json b/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json index 882342e23f..97079df64f 100644 --- a/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json +++ b/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json b/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json index 784a21b801..0f72e41b9f 100644 --- a/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json +++ b/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-05cb171b610bfb45f6228128a385cde8a5b86d7ca377a028004cc382e12faf41.json b/backend/.sqlx/query-7b5dc50b685afc7de0b99bd68e2217d660529c15b41860a4513f63d465bb9abb.json similarity index 63% rename from backend/.sqlx/query-05cb171b610bfb45f6228128a385cde8a5b86d7ca377a028004cc382e12faf41.json rename to backend/.sqlx/query-7b5dc50b685afc7de0b99bd68e2217d660529c15b41860a4513f63d465bb9abb.json index 5290fab4c4..7bd7c1a192 100644 --- a/backend/.sqlx/query-05cb171b610bfb45f6228128a385cde8a5b86d7ca377a028004cc382e12faf41.json +++ b/backend/.sqlx/query-7b5dc50b685afc7de0b99bd68e2217d660529c15b41860a4513f63d465bb9abb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO concurrency_counter(concurrency_id, job_uuids) \n VALUES ($1, '{}'::jsonb)", + "query": "INSERT INTO concurrency_counter(concurrency_id, job_uuids)\n VALUES ($1, '{}'::jsonb)", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "05cb171b610bfb45f6228128a385cde8a5b86d7ca377a028004cc382e12faf41" + "hash": "7b5dc50b685afc7de0b99bd68e2217d660529c15b41860a4513f63d465bb9abb" } diff --git a/backend/.sqlx/query-625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31.json b/backend/.sqlx/query-81ba1e003d3f6e8e7c74a4dd09b980888d71121082390f3fdb91269cd0c5d0b2.json similarity index 52% rename from backend/.sqlx/query-625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31.json rename to backend/.sqlx/query-81ba1e003d3f6e8e7c74a4dd09b980888d71121082390f3fdb91269cd0c5d0b2.json index 248f44b23a..83b4cc8e28 100644 --- a/backend/.sqlx/query-625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31.json +++ b/backend/.sqlx/query-81ba1e003d3f6e8e7c74a4dd09b980888d71121082390f3fdb91269cd0c5d0b2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n LEFT JOIN flow_version_lite \n ON flow_version_lite.id = flow_version.id\n WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1", + "query": "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow\n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n LEFT JOIN flow_version_lite\n ON flow_version_lite.id = flow_version.id\n WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1", "describe": { "columns": [ { @@ -19,5 +19,5 @@ null ] }, - "hash": "625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31" + "hash": "81ba1e003d3f6e8e7c74a4dd09b980888d71121082390f3fdb91269cd0c5d0b2" } diff --git a/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json b/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json index a60730197f..14d37809d5 100644 --- a/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json +++ b/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json b/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json index ec393b4218..acc6769652 100644 --- a/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json +++ b/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json b/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json index f8c8963a18..f38b293acb 100644 --- a/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json +++ b/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378.json b/backend/.sqlx/query-b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378.json deleted file mode 100644 index b495061906..0000000000 --- a/backend/.sqlx/query-b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE \n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n CASE\n -- flow step:\n WHEN flow_step_id IS NOT NULL THEN NULL\n -- completed:\n WHEN c.id IS NOT NULL THEN COALESCE(\n c.workflow_as_code_status || c.flow_status,\n c.workflow_as_code_status,\n c.flow_status\n )\n -- not completed:\n ELSE COALESCE(\n f.workflow_as_code_status || f.flow_status,\n f.workflow_as_code_status,\n f.flow_status\n )\n END AS \"flow_status: sqlx::types::Json>\",\n job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "completed", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "running", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "logs", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "mem_peak", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "flow_status: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 5, - "name": "log_offset", - "type_info": "Int4" - }, - { - "ordinal": 6, - "name": "created_by!", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "progress", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Text", - "Uuid", - "Bool", - "Bool" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null, - false, - null - ] - }, - "hash": "b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378" -} diff --git a/backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json b/backend/.sqlx/query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json similarity index 60% rename from backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json rename to backend/.sqlx/query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json index 8d51b798e9..4417c453f9 100644 --- a/backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json +++ b/backend/.sqlx/query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT \n j.id,\n j.kind AS \"kind: _\",\n COALESCE(s.path, f.path) AS \"script_path!\",\n COALESCE(s.hash, f.id) AS \"script_hash!: _\",\n COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS \"scheduled_for!: _\",\n args AS input,\n COALESCE(s.schema, f.schema) AS \"schema: _\"\n FROM v2_job j\n LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n LEFT JOIN v2_job_completed jc ON jc.id = j.id\n LEFT JOIN v2_job_queue jq ON jq.id = j.id\n WHERE j.id = ANY($1)\n AND j.workspace_id = $2\n AND COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL", + "query": "SELECT\n j.id,\n j.kind AS \"kind: _\",\n COALESCE(s.path, f.path) AS \"script_path!\",\n COALESCE(s.hash, f.id) AS \"script_hash!: _\",\n COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS \"scheduled_for!: _\",\n args AS input,\n COALESCE(s.schema, f.schema) AS \"schema: _\"\n FROM v2_job j\n LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n LEFT JOIN v2_job_completed jc ON jc.id = j.id\n LEFT JOIN v2_job_queue jq ON jq.id = j.id\n WHERE j.id = ANY($1)\n AND j.workspace_id = $2\n AND COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL", "describe": { "columns": [ { @@ -81,5 +81,5 @@ null ] }, - "hash": "ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9" + "hash": "b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a" } diff --git a/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json b/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json index b378d68f5b..4843d959c1 100644 --- a/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json +++ b/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c.json b/backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json similarity index 72% rename from backend/.sqlx/query-2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c.json rename to backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json index 6d916281eb..a76515fd7e 100644 --- a/backend/.sqlx/query-2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c.json +++ b/backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job \n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id \n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", + "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", "describe": { "columns": [ { @@ -38,5 +38,5 @@ true ] }, - "hash": "2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c" + "hash": "ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9" } diff --git a/backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json b/backend/.sqlx/query-cad2dfc50d57c2298cb0ec05ed3b42b9fd3edd05c314c9d96ba33cad37fe35c2.json similarity index 64% rename from backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json rename to backend/.sqlx/query-cad2dfc50d57c2298cb0ec05ed3b42b9fd3edd05c314c9d96ba33cad37fe35c2.json index 1ca60067d0..73bab032f1 100644 --- a/backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json +++ b/backend/.sqlx/query-cad2dfc50d57c2298cb0ec05ed3b42b9fd3edd05c314c9d96ba33cad37fe35c2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", + "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id)\n SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", "describe": { "columns": [], "parameters": { @@ -17,5 +17,5 @@ }, "nullable": [] }, - "hash": "fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa" + "hash": "cad2dfc50d57c2298cb0ec05ed3b42b9fd3edd05c314c9d96ba33cad37fe35c2" } diff --git a/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json b/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json index f550276925..912b9a53de 100644 --- a/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json +++ b/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json b/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json index 23b70ac4d2..79fe0873dd 100644 --- a/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json +++ b/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json b/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json index 1078957cf6..cdb8b928b3 100644 --- a/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json +++ b/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json b/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json index 90413de23c..a1ce88bd88 100644 --- a/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json +++ b/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836.json b/backend/.sqlx/query-f6e63a324664423353b56bccd9f3b322a3512ccfa86485e0f01664942ae04d0f.json similarity index 78% rename from backend/.sqlx/query-70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836.json rename to backend/.sqlx/query-f6e63a324664423353b56bccd9f3b322a3512ccfa86485e0f01664942ae04d0f.json index 9278984d51..4e21dac9a9 100644 --- a/backend/.sqlx/query-70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836.json +++ b/backend/.sqlx/query-f6e63a324664423353b56bccd9f3b322a3512ccfa86485e0f01664942ae04d0f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH uuid_table as (\n select unnest($4::uuid[]) as uuid\n )\n INSERT INTO v2_job_queue\n (id, workspace_id, scheduled_for, tag)\n (SELECT uuid, $1, $2, $3 FROM uuid_table) \n RETURNING id", + "query": "WITH uuid_table as (\n select unnest($4::uuid[]) as uuid\n )\n INSERT INTO v2_job_queue\n (id, workspace_id, scheduled_for, tag)\n (SELECT uuid, $1, $2, $3 FROM uuid_table)\n RETURNING id", "describe": { "columns": [ { @@ -21,5 +21,5 @@ false ] }, - "hash": "70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836" + "hash": "f6e63a324664423353b56bccd9f3b322a3512ccfa86485e0f01664942ae04d0f" } diff --git a/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json b/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json index 38f51ee611..091d205b40 100644 --- a/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json +++ b/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e95a7b9d82..bd4a7009d7 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -776,9 +776,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687bc16bc431a8533fe0097c7f0182874767f920989d7260950172ae8e3c4465" +checksum = "b68c2194a190e1efc999612792e25b1ab3abfefe4306494efaaabc25933c0cbe" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -835,9 +835,9 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "1.75.0" +version = "1.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c27b0a5c8f9f8db35929bde0612d738d8c872ab887232579c61d73dba7a5d17" +checksum = "837b4b181a1306a5a67d85218fa7d7298e340041d9bd81ae6919b9ec4f86ca97" dependencies = [ "aws-credential-types", "aws-runtime", @@ -857,9 +857,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.75.0" +version = "1.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36d2a4d2753a2304cc0a86cff15581179c420c7b3054c0cbc884a2e497e79ae5" +checksum = "64bf26698dd6d238ef1486bdda46f22a589dc813368ba868dc3d94c8d27b56ba" dependencies = [ "aws-credential-types", "aws-runtime", @@ -879,9 +879,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.76.0" +version = "1.77.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719f007d6914388e2f14943c663447cfc44bb9d0478bb96df891258ba5145f82" +checksum = "09cd07ed1edd939fae854a22054299ae3576500f4e0fadc560ca44f9c6ea1664" dependencies = [ "aws-credential-types", "aws-runtime", @@ -901,9 +901,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.77.0" +version = "1.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efcc3dbb630f3e0d16a4f79fcd4b3fb1c59cb89f1031c46e44976f640b5de27" +checksum = "37f7766d2344f56d10d12f3c32993da36d78217f32594fe4fb8e57a538c1cdea" dependencies = [ "aws-credential-types", "aws-runtime", @@ -14676,7 +14676,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "axum", @@ -14728,7 +14728,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "argon2", @@ -14839,7 +14839,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.507.1" +version = "1.508.0" dependencies = [ "base64 0.22.1", "chrono", @@ -14854,7 +14854,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.507.1" +version = "1.508.0" dependencies = [ "chrono", "serde", @@ -14867,7 +14867,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "serde", @@ -14881,7 +14881,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "async-stream", @@ -14934,6 +14934,7 @@ dependencies = [ "sqlx", "strum 0.27.1", "strum_macros 0.27.1", + "sysinfo", "systemstat", "tar", "tempfile", @@ -14959,7 +14960,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.507.1" +version = "1.508.0" dependencies = [ "regex", "serde", @@ -14973,7 +14974,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "bytes", @@ -14996,7 +14997,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.507.1" +version = "1.508.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15008,7 +15009,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.507.1" +version = "1.508.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15017,7 +15018,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "lazy_static", @@ -15029,7 +15030,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "serde_json", @@ -15041,7 +15042,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "gosyn", @@ -15053,7 +15054,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "lazy_static", @@ -15065,7 +15066,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "serde_json", @@ -15077,7 +15078,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "nu-parser", @@ -15088,7 +15089,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15099,7 +15100,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15111,7 +15112,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "async-recursion", @@ -15134,7 +15135,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15151,7 +15152,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "lazy_static", @@ -15165,7 +15166,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "lazy_static", @@ -15183,7 +15184,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -15207,7 +15208,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "serde_json", @@ -15217,7 +15218,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "async-recursion", @@ -15250,7 +15251,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.507.1" +version = "1.508.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15260,7 +15261,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.507.1" +version = "1.508.0" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a0a79b8385..2937431719 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.507.1" +version = "1.508.0" authors.workspace = true edition.workspace = true @@ -32,7 +32,7 @@ members = [ ] [workspace.package] -version = "1.507.1" +version = "1.508.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -267,6 +267,7 @@ google-cloud-googleapis = {version = "0.16.1", features = ["pubsub"]} # TODO: remove once deno fixes the issue on their end # https://github.com/denoland/deno/issues/28557 winapi = { version = "0.3.9", features = ["sysinfoapi"] } +sysinfo = { version = "0.32.1" } swc_common = "=0.37.5" swc_ecma_parser = "=0.149.1" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 653071700b..d545254b2e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -3cccbe81494f7edd113dce2812278b00c91160c0 \ No newline at end of file +773a959358d0dc30255621a584fc2b8e2dd41582 diff --git a/backend/migrations/20250717135503_assets_refactor.down.sql b/backend/migrations/20250717135503_assets_refactor.down.sql new file mode 100644 index 0000000000..76780e9200 --- /dev/null +++ b/backend/migrations/20250717135503_assets_refactor.down.sql @@ -0,0 +1,7 @@ +-- Add 'variable' kind back +ALTER TABLE asset ALTER column kind TYPE VARCHAR; +DROP TYPE asset_kind; +CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource', 'variable'); +ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND; + +ALTER TABLE script DROP COLUMN assets; diff --git a/backend/migrations/20250717135503_assets_refactor.up.sql b/backend/migrations/20250717135503_assets_refactor.up.sql new file mode 100644 index 0000000000..bd17555a9c --- /dev/null +++ b/backend/migrations/20250717135503_assets_refactor.up.sql @@ -0,0 +1,8 @@ +ALTER TABLE script ADD COLUMN assets jsonb; + +-- Remove 'variable' kind +DELETE FROM asset WHERE kind = 'variable'; +ALTER TABLE asset ALTER column kind TYPE VARCHAR; +DROP TYPE asset_kind; +CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource'); +ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND; \ No newline at end of file diff --git a/backend/parsers/windmill-parser-py/src/asset_parser.rs b/backend/parsers/windmill-parser-py/src/asset_parser.rs index d6f16824c7..a96057af1e 100644 --- a/backend/parsers/windmill-parser-py/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-py/src/asset_parser.rs @@ -57,27 +57,27 @@ impl AssetsFinder { }) .ok_or(())?; + use AssetKind::*; let (kind, access_type, arg) = match ident.as_str() { - "load_s3_file" => (AssetKind::S3Object, Some(R), Arg::Pos(0)), - "load_s3_file_reader" => (AssetKind::S3Object, Some(R), Arg::Pos(0)), - "write_s3_file" => (AssetKind::S3Object, Some(W), Arg::Pos(0)), - "get_resource" => (AssetKind::Resource, None, Arg::Pos(0)), - "set_resource" => (AssetKind::Resource, Some(W), Arg::Named("path")), - "get_boto3_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), - "get_polars_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), - "get_duckdb_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), - "get_variable" => (AssetKind::Variable, Some(R), Arg::Pos(0)), - "set_variable" => (AssetKind::Variable, Some(W), Arg::Pos(0)), + "load_s3_file" => (S3Object, Some(R), Arg::Pos(0, "s3object")), + "load_s3_file_reader" => (S3Object, Some(R), Arg::Pos(0, "s3object")), + "write_s3_file" => (S3Object, Some(W), Arg::Pos(0, "s3object")), + "get_resource" => (Resource, None, Arg::Pos(0, "path")), + "set_resource" => (Resource, Some(W), Arg::Pos(0, "path")), + "get_boto3_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), + "get_polars_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), + "get_duckdb_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), _ => return Err(()), }; let arg_val = match arg { - Arg::Pos(i) => node.args.get(i), - Arg::Named(name) => node - .keywords - .iter() - .find(|kw| kw.arg.as_deref() == Some(name)) - .map(|kw| &kw.value), + Arg::Pos(i, name) => node.args.get(i).or_else(|| { + // Get arg by name + node.keywords + .iter() + .find(|kw| kw.arg.as_deref() == Some(name)) + .map(|kw| &kw.value) + }), }; match arg_val { @@ -93,6 +93,6 @@ impl AssetsFinder { } enum Arg { - Pos(usize), - Named(&'static str), + // Positional arguments in python can also be used by their name + Pos(usize, &'static str), } diff --git a/backend/parsers/windmill-parser-ts/src/asset_parser.rs b/backend/parsers/windmill-parser-ts/src/asset_parser.rs index dcd6681f5f..6b81272ec1 100644 --- a/backend/parsers/windmill-parser-ts/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-ts/src/asset_parser.rs @@ -84,8 +84,6 @@ impl AssetsFinder { "denoS3LightClientSettings" => (AssetKind::Resource, None, 0), "duckdbConnectionSettings" => (AssetKind::Resource, None, 0), "polarsConnectionSettings" => (AssetKind::Resource, None, 0), - "getVariable" => (AssetKind::Variable, Some(R), 0), - "setVariable" => (AssetKind::Variable, Some(W), 0), _ => return Err(()), }; diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 893843342b..6857e8d68e 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -15,7 +15,6 @@ use AssetUsageAccessType::*; pub enum AssetKind { S3Object, Resource, - Variable, } #[derive(Serialize)] @@ -57,8 +56,6 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> { Some((AssetKind::Resource, &s[6..])) } else if s.starts_with("$res:") { Some((AssetKind::Resource, &s[5..])) - } else if s.starts_with("var://") { - Some((AssetKind::Variable, &s[6..])) } else { None } diff --git a/backend/substitute_ee_code.sh b/backend/substitute_ee_code.sh index a1819c75b2..3ea73e16cc 100755 --- a/backend/substitute_ee_code.sh +++ b/backend/substitute_ee_code.sh @@ -99,4 +99,4 @@ if [ "$MOVE_NEW_FILES" == "YES" ]; then echo "File moved '${ce_file}' -->> '${ee_file}'" fi done -fi \ No newline at end of file +fi diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 28bce55024..65263152dc 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1147,7 +1147,7 @@ async fn test_deno_flow(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, } .into(), stop_after_if: Default::default(), @@ -1191,7 +1191,7 @@ async fn test_deno_flow(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, } .into(), stop_after_if: Default::default(), @@ -1321,7 +1321,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), @@ -1376,7 +1376,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), stop_after_all_iters_if: Default::default(), @@ -1416,7 +1416,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), @@ -1482,7 +1482,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), stop_after_all_iters_if: Default::default(), diff --git a/backend/update_sqlx.sh b/backend/update_sqlx.sh index 6e64115ce0..d8bc473f29 100755 --- a/backend/update_sqlx.sh +++ b/backend/update_sqlx.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Default directory EE_DIR="../windmill-ee-private" @@ -32,4 +32,4 @@ if [[ "$(uname)" == "Darwin" ]]; then sed -i '' 's/^#samael = { version="0.0.14", features = \["xmlsec"\] }/samael = { version="0.0.14", features = ["xmlsec"] }/' Cargo.toml # Comment out the git-based samael dependency sed -i '' 's/^\(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/# \1/' Cargo.toml -fi \ No newline at end of file +fi diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 9e794fa7f0..c7b16140fc 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.507.1 + version: 1.508.0 title: Windmill API contact: @@ -2792,6 +2792,22 @@ paths: schema: type: string + /tokens/list/scopes: + get: + summary: list of available scopes + operationId: ListAvailableScopes + tags: + - token + responses: + "200": + description: list of available scopes + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ScopeDomain" + /users/tokens/create: post: summary: create token @@ -13856,20 +13872,27 @@ components: type: boolean on_behalf_of_email: type: string - fallback_access_types: + assets: type: array items: type: object - required: [path, kind, access_type] + required: + - path + - kind properties: path: type: string kind: type: string - enum: [s3object, resource] + enum: + - s3object + - resource access_type: type: string enum: [r, w, rw] + alt_access_type: + type: string + enum: [r, w, rw] required: - path @@ -16771,6 +16794,40 @@ components: - edited_at - execution_mode + ScopeDefinition: + type: object + properties: + value: + type: string + label: + type: string + description: + type: string + nullable: true + requires_resource_path: + type: boolean + required: + - value + - label + - requires_resource_path + + ScopeDomain: + type: object + properties: + name: + type: string + description: + type: string + nullable: true + scopes: + type: array + items: + $ref: "#/components/schemas/ScopeDefinition" + required: + - name + - scopes + + ListableRawApp: type: object properties: @@ -17590,7 +17647,6 @@ components: enum: - s3object - resource - - variable Asset: type: object properties: diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index ad4020a671..188e6e3a81 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -13,7 +13,7 @@ use crate::{ db::{ApiAuthed, DB}, resources::get_resource_value_interpolated_internal, users::{require_owner_of_path, OptAuthed}, - utils::WithStarredInfoQuery, + utils::{check_scopes, WithStarredInfoQuery}, webhook_util::{WebhookMessage, WebhookShared}, HTTP_CLIENT, }; @@ -444,6 +444,7 @@ async fn get_app( Query(query): Query, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("apps:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let app_o = if query.with_starred_info.unwrap_or(false) { @@ -491,6 +492,7 @@ async fn get_app_lite( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("apps:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let app_o = sqlx::query_as::<_, AppWithLastVersion>( @@ -518,6 +520,7 @@ async fn get_app_w_draft( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("apps:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let app_o = sqlx::query_as::<_, AppWithLastVersionAndDraft>( @@ -547,6 +550,8 @@ async fn get_app_history( Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("apps:read:{}", &path))?; let mut tx = user_db.begin(&authed).await?; let query_result = sqlx::query!( "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg @@ -554,7 +559,7 @@ async fn get_app_history( WHERE a.workspace_id = $1 AND a.path = $2 ORDER BY created_at DESC", w_id, - path.to_path(), + path, ).fetch_all(&mut *tx).await?; tx.commit().await?; @@ -574,6 +579,8 @@ async fn get_latest_version( Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("apps:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let row = sqlx::query!( "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg @@ -581,7 +588,7 @@ async fn get_latest_version( WHERE a.workspace_id = $1 AND a.path = $2 ORDER BY created_at DESC", w_id, - path.to_path(), + path, ).fetch_optional(&mut *tx).await?; tx.commit().await?; @@ -609,17 +616,19 @@ async fn update_app_history( .fetch_optional(&mut *tx) .await?; - if app_path.is_none() { + let Some(app_path) = app_path else { tx.commit().await?; return Err(Error::NotFound( format!("App with ID {app_id} not found").to_string(), )); - } + }; + + check_scopes(&authed, || format!("apps:write:{}", &app_path))?; sqlx::query!( "INSERT INTO deployment_metadata (workspace_id, path, app_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET deployment_msg = $4", w_id, - app_path.unwrap(), + app_path, app_version, app_history_update.deployment_msg, ) @@ -664,6 +673,9 @@ async fn get_app_by_id( tx.commit().await?; let app = not_found_if_none(app_o, "App", id.to_string())?; + + check_scopes(&authed, || format!("apps:read:{}", &app.path))?; + Ok(Json(app)) } @@ -756,6 +768,7 @@ async fn get_secret_id( Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("apps:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let id_o = sqlx::query_scalar!( @@ -861,6 +874,8 @@ async fn create_app_raw<'a>( ) .await?; + check_scopes(&authed, || format!("apps:write:{}", path))?; + webhook.send_message( w_id.clone(), WebhookMessage::CreateApp { workspace: w_id, path: path.clone() }, @@ -899,6 +914,8 @@ async fn create_app( Json(app): Json, ) -> Result<(StatusCode, String)> { let path = app.path.clone(); + check_scopes(&authed, || format!("apps:write:{}", &path))?; + let (new_tx, _path, _id) = create_app_internal(authed, db, user_db, &w_id, false, app).await?; new_tx.commit().await?; @@ -1101,6 +1118,7 @@ async fn delete_app( Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("apps:write:{}", path))?; if path == "g/all/setup_app" && w_id == "admins" { return Err(Error::BadRequest( @@ -1184,6 +1202,7 @@ async fn update_app( ) -> Result { // create_app_internal(authed, user_db, db, &w_id, &mut app).await?; let path = path.to_path(); + check_scopes(&authed, || format!("apps:write:{}", path))?; let opath = path.to_string(); let (new_tx, npath, _v_id) = update_app_internal(authed, db, user_db, &w_id, path, false, ns).await?; @@ -1210,6 +1229,7 @@ async fn update_app_raw<'a>( multipart: Multipart, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("apps:write:{}", path))?; let opath = path.to_string(); let (npath, _id) = process_app_multipart!( authed, @@ -2421,10 +2441,9 @@ async fn exists_app( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { - let path = path.to_path(); let exists = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)", - path, + path.to_path(), w_id ) .fetch_one(&db) diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index a61e2ee72b..a1ccc69769 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -22,6 +22,7 @@ use tokio::sync::RwLock; use windmill_common::{ auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, TOKEN_PREFIX_LEN}, + error::Error, jwt, users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL}, }; @@ -218,7 +219,9 @@ impl AuthCache { folders, scopes: None, username_override, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), }) } else { let groups = vec![name.to_string()]; @@ -240,7 +243,9 @@ impl AuthCache { folders, scopes: None, username_override, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), }) } } else { @@ -303,7 +308,9 @@ impl AuthCache { folders, scopes, username_override, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), }) } None if super_admin => Some(ApiAuthed { @@ -315,7 +322,9 @@ impl AuthCache { folders: vec![], scopes, username_override, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), }), None => None, } @@ -491,6 +500,33 @@ where } } +pub fn transform_old_scope_to_new_scope(scopes: Option<&mut Vec>) { + if let Some(scopes) = scopes { + for scope in scopes.iter_mut() { + if scope.starts_with("run:") { + let (_, part_scope) = scope.split_once(":").unwrap(); + + if let Some((kind, path)) = part_scope.split_once("/") { + //appending a 's' as runnable kind is singular while new scope format expect it to be plural + *scope = format!("jobs:run:{}s:{}", kind, path); + } + } else if scope.starts_with("jobs:") { + // Map old jobs scopes to new format + let new_scope = match scope.as_str() { + "jobs:listjobs" => "jobs:read", + "jobs:runscript" => "jobs:run:scripts", + "jobs:runflow" => "jobs:run:flows", + "jobs:resumeflow" => "jobs:run:flows", + "jobs:deletejob" => "jobs:write", + _ => continue, + }; + + *scope = new_scope.to_string(); + } + } + } +} + fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option { let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { Some(path_vec[3].to_owned()) @@ -517,7 +553,7 @@ impl FromRequestParts for ApiAuthed where S: Send + Sync, { - type Rejection = (StatusCode, String); + type Rejection = Error; async fn from_request_parts( parts: &mut Parts, @@ -538,7 +574,6 @@ where } else { extract_token(parts, state).await }; - if let Some(token) = token_o { if let Ok(Extension(cache)) = Extension::>::from_request_parts(parts, state).await @@ -551,21 +586,22 @@ where let path_vec: Vec<&str> = original_uri.path().split("/").collect(); let workspace_id = maybe_get_workspace_id_from_path(&path_vec); - if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await { - if authed.scopes.as_ref().is_some_and(|scopes| { - scopes - .iter() - .any(|s| s.starts_with("jobs:") || s.starts_with("run:")) - }) && (path_vec.len() < 3 - || (path_vec[4] != "jobs" && path_vec[4] != "jobs_u")) - { - BRUTE_FORCE_COUNTER.increment().await; - return Err(( - StatusCode::UNAUTHORIZED, - format!("Unauthorized scoped token: {:?}", authed.scopes), - )); - } + if let Some(mut authed) = cache.get_authed(workspace_id.clone(), &token).await { + if authed.scopes.is_some() { + transform_old_scope_to_new_scope(authed.scopes.as_mut()); + let path = original_uri.path(); + let method = parts.method.as_str(); + + if let Err(err) = crate::scopes::check_scopes_for_route( + authed.scopes.as_deref(), + path, + method, + ) { + BRUTE_FORCE_COUNTER.increment().await; + return Err(err); + } + } parts.extensions.insert(authed.clone()); Span::current().record("username", &authed.username.as_str()); @@ -579,7 +615,7 @@ where } } BRUTE_FORCE_COUNTER.increment().await; - Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) + Err(Error::NotAuthorized("Unauthorized".to_string())) } } diff --git a/backend/windmill-api/src/concurrency_groups.rs b/backend/windmill-api/src/concurrency_groups.rs index 4135c3729c..ba85043a73 100644 --- a/backend/windmill-api/src/concurrency_groups.rs +++ b/backend/windmill-api/src/concurrency_groups.rs @@ -1,9 +1,11 @@ -use crate::db::{ApiAuthed, DB}; -use crate::jobs::{ - filter_list_completed_query, filter_list_queue_query, Job, ListCompletedQuery, ListQueueQuery, - UnifiedJob, +use crate::{ + db::{ApiAuthed, DB}, + jobs::{ + filter_list_completed_query, filter_list_queue_query, Job, ListCompletedQuery, + ListQueueQuery, UnifiedJob, + }, + utils::check_scopes, }; -use crate::users::check_scopes; use axum::extract::Path; use axum::routing::{delete, get}; use axum::{extract::Query, Extension, Json}; @@ -144,7 +146,7 @@ async fn get_concurrent_intervals( Query(iq): Query, Query(lq): Query, ) -> JsonResult { - check_scopes(&authed, || format!("jobs:listjobs"))?; + check_scopes(&authed, || format!("jobs:read"))?; if lq.success.is_some() && lq.running.is_some_and(|x| x) { return Err(error::Error::BadRequest( diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index aa7bf29475..f9d7827341 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -8,15 +8,15 @@ use std::collections::HashMap; -use crate::db::ApiAuthed; -use crate::triggers::{ - get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail, -}; use crate::utils::WithStarredInfoQuery; use crate::{ - db::DB, + db::{ApiAuthed, DB}, schedule::clear_schedule, + triggers::{ + get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail, + }, users::{maybe_refresh_folders, require_owner_of_path}, + utils::check_scopes, webhook_util::{WebhookMessage, WebhookShared}, HTTP_CLIENT, }; @@ -333,6 +333,8 @@ async fn list_paths_from_workspace_runnable( Extension(user_db): Extension, Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>, ) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let runnables = sqlx::query_scalar!( r#"SELECT f.path @@ -340,7 +342,7 @@ async fn list_paths_from_workspace_runnable( JOIN flow f ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, - path.to_path(), + path, matches!(runnable_kind, RunnableKind::Flow), w_id ) @@ -358,6 +360,7 @@ async fn create_flow( Path(w_id): Path, Json(nf): Json, ) -> Result<(StatusCode, String)> { + check_scopes(&authed, || format!("flows:write:{}", nf.path))?; if *CLOUD_HOSTED { let nb_flows = sqlx::query_scalar!("SELECT COUNT(*) FROM flow WHERE workspace_id = $1", &w_id) @@ -579,6 +582,7 @@ async fn get_flow_history( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult> { let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let flows = sqlx::query_as!( @@ -603,6 +607,7 @@ async fn get_latest_version( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult> { let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let version = sqlx::query_as!( @@ -627,6 +632,7 @@ async fn get_flow_version( Path((w_id, version, path)): Path<(String, i64, StripPath)>, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let flow = sqlx::query_as::<_, Flow>( @@ -660,6 +666,7 @@ async fn update_flow_history( Json(history_update): Json, ) -> Result<()> { let path = path.to_path(); + check_scopes(&authed, || format!("flows:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; let path_o = sqlx::query_scalar!( "SELECT flow.path FROM flow @@ -701,6 +708,8 @@ async fn update_flow( Path((w_id, flow_path)): Path<(String, StripPath)>, Json(nf): Json, ) -> Result { + let flow_path = flow_path.to_path(); + check_scopes(&authed, || format!("flows:write:{}", flow_path))?; #[cfg(not(feature = "enterprise"))] if nf .value @@ -714,7 +723,6 @@ async fn update_flow( )); } - let flow_path = flow_path.to_path(); let authed = maybe_refresh_folders(&flow_path, &w_id, authed, &db).await; let mut tx = user_db.clone().begin(&authed).await?; @@ -1037,6 +1045,7 @@ async fn get_flow_by_path( Query(query): Query, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let flow_o = if query.with_starred_info.unwrap_or(false) { @@ -1104,10 +1113,11 @@ async fn get_flow_by_path_w_draft( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let flow_o = sqlx::query_as::<_, FlowWDraft>( - "SELECT flow.path, flow.summary, flow,description, flow_version.schema, flow_version.value, flow.extra_perms, flow.draft_only, flow.ws_error_handler_muted, flow.dedicated_worker, draft.value as draft, flow.tag, flow.visible_to_runner_only, flow.on_behalf_of_email + "SELECT flow.path, flow.summary, flow.description, flow_version.schema, flow_version.value, flow.extra_perms, flow.draft_only, flow.ws_error_handler_muted, flow.dedicated_worker, draft.value as draft, flow.tag, flow.visible_to_runner_only, flow.on_behalf_of_email FROM flow LEFT JOIN draft ON flow.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'flow' @@ -1157,6 +1167,7 @@ async fn archive_flow_by_path( Json(archived): Json, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("flows:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; sqlx::query!( @@ -1233,6 +1244,7 @@ async fn delete_flow_by_path( Query(query): Query, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("flows:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; sqlx::query!( @@ -1379,7 +1391,7 @@ mod tests { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }), stop_after_if: Some(StopAfterIf { expr: "foo = 'bar'".to_string(), diff --git a/backend/windmill-api/src/folders.rs b/backend/windmill-api/src/folders.rs index 1e4d2e09c0..7dc5951bda 100644 --- a/backend/windmill-api/src/folders.rs +++ b/backend/windmill-api/src/folders.rs @@ -8,12 +8,11 @@ use std::sync::Arc; -use crate::db::ApiAuthed; - use crate::{ auth::AuthCache, - db::DB, + db::{ApiAuthed, DB}, users::Tokened, + utils::check_scopes, webhook_util::{WebhookMessage, WebhookShared}, }; use axum::{ @@ -419,6 +418,7 @@ async fn get_folder( Extension(user_db): Extension, Path((w_id, name)): Path<(String, String)>, ) -> JsonResult { + check_scopes(&authed, || format!("folders:read:f/{}", name))?; let mut tx = user_db.begin(&authed).await?; let folder = not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; @@ -457,6 +457,7 @@ async fn get_folder_usage( Extension(user_db): Extension, Path((w_id, name)): Path<(String, String)>, ) -> JsonResult { + check_scopes(&authed, || format!("folders:read:f/{}", name))?; let mut tx = user_db.begin(&authed).await?; let scripts = sqlx::query_scalar!( diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs index 27607478db..b6d5fff7f0 100644 --- a/backend/windmill-api/src/http_triggers.rs +++ b/backend/windmill-api/src/http_triggers.rs @@ -2,16 +2,17 @@ use crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs}; #[cfg(feature = "parquet")] use crate::job_helpers_oss::get_workspace_s3_resource; -use crate::resources::try_get_resource_from_db_as; -use crate::trigger_helpers::{ - get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, RunnableId, -}; -use crate::utils::{non_empty_str, ExpiringCacheEntry}; use crate::{ auth::{AuthCache, OptTokened}, db::{ApiAuthed, DB}, + resources::try_get_resource_from_db_as, + trigger_helpers::{ + get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, RunnableId, + }, users::fetch_api_authed, + utils::{check_scopes, non_empty_str, ExpiringCacheEntry}, }; + use anyhow::anyhow; use axum::response::Response; use axum::{ @@ -253,8 +254,9 @@ async fn get_trigger( Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> error::JsonResult { - let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); + check_scopes(&authed, || format!("http_triggers:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; let trigger = sqlx::query_as!( HttpTrigger, r#" @@ -333,6 +335,10 @@ async fn create_trigger_inner( new_http_trigger: &NewTrigger, route_path_key: &str, ) -> WindmillResult<()> { + check_scopes(&authed, || { + format!("http_triggers:write:{}", &new_http_trigger.path) + })?; + sqlx::query!( r#" INSERT INTO http_trigger ( @@ -577,6 +583,7 @@ async fn update_trigger( Json(ct): Json, ) -> WindmillResult { let path = path.to_path(); + check_scopes(&authed, || format!("http_triggers:write:{}", path))?; if *CLOUD_HOSTED && (ct.is_static_website || ct.static_asset_config.is_some()) { return Err(error::Error::BadRequest( @@ -761,8 +768,9 @@ async fn delete_trigger( Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> WindmillResult { - require_admin(authed.is_admin, &authed.username)?; let path = path.to_path(); + check_scopes(&authed, || format!("http_triggers:write:{}", path))?; + require_admin(authed.is_admin, &authed.username)?; let mut tx = user_db.begin(&authed).await?; sqlx::query!( "DELETE FROM http_trigger diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index a31689a8cf..899f97e03d 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -35,19 +35,15 @@ use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR}; use windmill_common::scripts::PREVIEW_IS_CODEBASE_HASH; use windmill_common::variables::get_workspace_key; -use crate::add_webhook_allowed_origin; -use crate::auth::{OptTokened, Tokened}; -use crate::concurrency_groups::join_concurrency_key; -use crate::db::ApiAuthed; - -use crate::trigger_helpers::RunnableId; -use crate::users::get_scope_tags; -use crate::utils::content_plain; use crate::{ + add_webhook_allowed_origin, args::{self, RawWebhookArgs}, - db::DB, - users::{check_scopes, require_owner_of_path, OptAuthed}, - utils::require_super_admin, + auth::{OptTokened, Tokened}, + concurrency_groups::join_concurrency_key, + db::{ApiAuthed, DB}, + trigger_helpers::RunnableId, + users::{get_scope_tags, require_owner_of_path, OptAuthed}, + utils::{check_scopes, content_plain, require_super_admin}, }; use anyhow::Context; use axum::{ @@ -722,7 +718,7 @@ macro_rules! get_job_query { "SELECT \ {table}.id, {table}.workspace_id, parent_job, v2_job.created_by, v2_job.created_at, started_at, v2_job.runnable_id as script_hash, v2_job.runnable_path as script_path, \ CASE WHEN args is null THEN NULL - WHEN pg_column_size(args) < 90000 THEN + WHEN pg_column_size(args) < 90000 THEN CASE WHEN jsonb_typeof(args) = 'object' THEN args ELSE jsonb_build_object('value', args) END @@ -732,7 +728,7 @@ macro_rules! get_job_query { {flow} as raw_flow, flow_step_id IS NOT NULL AS is_flow_step, script_lang as language, \ {lock} as raw_lock, permissioned_as_email as email, visible_to_owner, memory_peak as mem_peak, v2_job.tag, v2_job.priority, preprocessed, worker,\ {additional_fields} \ - FROM {table} + FROM {table} INNER JOIN v2_job ON v2_job.id = {table}.id \ {additional_joins} \ LEFT JOIN job_logs ON {table}.id = job_id \ @@ -1141,8 +1137,8 @@ async fn get_job_logs( let record = sqlx::query!( "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index - FROM v2_as_completed_job - LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id + FROM v2_as_completed_job + LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", id, w_id, @@ -1185,8 +1181,8 @@ async fn get_job_logs( } else { let text = sqlx::query!( "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index - FROM v2_as_queue - LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id + FROM v2_as_queue + LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", id, w_id, @@ -1244,7 +1240,7 @@ async fn get_args( .flatten(); let record = sqlx::query!( "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\" - FROM v2_as_completed_job + FROM v2_as_completed_job WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", id, &w_id, @@ -1784,7 +1780,6 @@ async fn list_filtered_job_uuids( Query(lq): Query, ) -> error::JsonResult> { require_admin(authed.is_admin, &authed.username)?; - check_scopes(&authed, || format!("jobs:listjobs"))?; let mut sqlb = list_completed_jobs_query( w_id.as_str(), @@ -1949,8 +1944,6 @@ async fn list_jobs( Query(pagination): Query, Query(lq): Query, ) -> error::JsonResult> { - check_scopes(&authed, || format!("jobs:listjobs"))?; - let limit = pagination.per_page.unwrap_or(1000); let (per_page, offset) = paginate(pagination); let lqc = lq.clone(); @@ -2026,16 +2019,15 @@ pub async fn resume_suspended_flow_as_owner( Path((_w_id, flow_id)): Path<(String, Uuid)>, QueryOrBody(value): QueryOrBody, ) -> error::Result { - check_scopes(&authed, || format!("jobs:resumeflow"))?; - let value = value.unwrap_or(serde_json::Value::Null); let mut tx = db.begin().await?; let (flow, job_id) = get_suspended_flow_info(flow_id, &mut tx).await?; - require_owner_of_path( - &authed, - &flow.script_path.clone().unwrap_or_else(|| String::new()), - )?; + let flow_path = flow.script_path.as_deref().unwrap_or_else(|| ""); + require_owner_of_path(&authed, flow_path)?; + check_scopes(&authed, || format!("jobs:run:flows:{}", flow_path))?; + + let value = value.unwrap_or(serde_json::Value::Null); insert_resume_job( 0, @@ -3288,17 +3280,18 @@ async fn batch_rerun_jobs( Extension(user_db): Extension, Path(w_id): Path, Json(body): Json, -) -> Response { +) -> error::Result { + check_scopes(&authed, || format!("jobs:run"))?; let stream = batch_rerun_jobs_inner(authed, db, user_db, w_id, body); let body = axum::body::Body::from_stream(stream.map(Result::<_, std::convert::Infallible>::Ok)); - Response::builder() + Ok(Response::builder() .status(201) .header("Content-Type", "text/event-stream") .header("Cache-Control", "no-cache") .body(body) - .unwrap() + .unwrap()) } fn batch_rerun_jobs_inner( @@ -3312,7 +3305,7 @@ fn batch_rerun_jobs_inner( tokio::spawn(async move { let mut job_stream = sqlx::query_as!( BatchReRunQueryReturnType, - r#"SELECT + r#"SELECT j.id, j.kind AS "kind: _", COALESCE(s.path, f.path) AS "script_path!", @@ -3507,8 +3500,9 @@ pub async fn run_flow_by_path_inner( ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; + let flow_path = flow_path.to_path(); - check_scopes(&authed, || format!("run:flow/{flow_path}"))?; + check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; let mut tx = user_db.clone().begin(&authed).await?; @@ -3633,7 +3627,7 @@ pub async fn restart_flow( let flow_path = completed_job .script_path .with_context(|| "No flow path set for completed flow job")?; - check_scopes(&authed, || format!("run:flow/{flow_path}"))?; + check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; let ehm = HashMap::new(); let push_args = completed_job @@ -3713,8 +3707,7 @@ pub async fn run_script_by_path_inner( check_license_key_valid().await?; let script_path = script_path.to_path(); - - check_scopes(&authed, || format!("run:script/{script_path}"))?; + check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; let mut tx = user_db.clone().begin(&authed).await?; let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = @@ -3787,6 +3780,11 @@ pub async fn run_workflow_as_code( Query(wkflow_query): Query, Json(task): Json, ) -> error::Result<(StatusCode, String)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &authed).await?; + check_scopes(&authed, || format!("jobs:run"))?; + let mut i = 1; if *CLOUD_HOSTED { @@ -3794,10 +3792,6 @@ pub async fn run_workflow_as_code( i += 1; } - #[cfg(feature = "enterprise")] - check_license_key_valid().await?; - check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &authed).await?; - if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); i += 1; @@ -3927,7 +3921,7 @@ pub async fn run_workflow_as_code( VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3)) ON CONFLICT (id) DO UPDATE SET workflow_as_code_status = JSONB_SET( - COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB), + COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB), array[$2], $3 )", @@ -4364,6 +4358,9 @@ pub async fn run_wait_result_job_by_path_get( #[cfg(feature = "enterprise")] check_license_key_valid().await?; + let script_path = script_path.to_path(); + check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; + if method == http::Method::HEAD { return Ok(Json(serde_json::json!("")).into_response()); } @@ -4380,8 +4377,6 @@ pub async fn run_wait_result_job_by_path_get( let mut args = args.process_args(&authed, &db, &w_id, None).await?; args.body = args::Body::HashMap(payload_args); - let script_path = script_path.to_path(); - let args = args .to_args_from_runnable( &db, @@ -4392,7 +4387,6 @@ pub async fn run_wait_result_job_by_path_get( .await?; check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; - check_scopes(&authed, || format!("run:script/{script_path}"))?; let mut tx = user_db.clone().begin(&authed).await?; let (job_payload, tag, delete_after_use, timeout, on_behalf_authed) = @@ -4531,9 +4525,10 @@ pub async fn run_wait_result_script_by_path_internal( w_id: String, args: PushArgsOwned, ) -> error::Result { - check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; let script_path = script_path.to_path(); - check_scopes(&authed, || format!("run:script/{script_path}"))?; + check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; + + check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; let mut tx = user_db.clone().begin(&authed).await?; let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = @@ -4640,7 +4635,7 @@ pub async fn run_wait_result_script_by_hash( if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); } - check_scopes(&authed, || format!("run:script/{path}"))?; + check_scopes(&authed, || format!("jobs:run:scripts:{path}"))?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; @@ -4746,7 +4741,7 @@ pub async fn run_wait_result_flow_by_path_internal( check_queue_too_long(&db, run_query.queue_limit).await?; let flow_path = flow_path.to_path(); - check_scopes(&authed, || format!("run:flow/{flow_path}"))?; + check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; let scheduled_for = run_query.get_scheduled_for(&db).await?; @@ -4830,8 +4825,6 @@ async fn run_preview_script( ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; - - check_scopes(&authed, || format!("jobs:runscript"))?; if authed.is_operator { return Err(error::Error::NotAuthorized( "Operators cannot run preview jobs for security reasons".to_string(), @@ -4901,7 +4894,6 @@ async fn run_bundle_preview_script( ) -> error::Result<(StatusCode, String)> { use windmill_common::scripts::PREVIEW_IS_TAR_CODEBASE_HASH; - check_scopes(&authed, || format!("jobs:runscript"))?; if authed.is_operator { return Err(error::Error::NotAuthorized( "Operators cannot run preview jobs for security reasons".to_string(), @@ -5078,6 +5070,7 @@ async fn run_dependencies_job( Path(w_id): Path, Json(req): Json, ) -> error::Result { + check_scopes(&authed, || format!("jobs:run"))?; if authed.is_operator { return Err(error::Error::NotAuthorized( "Operators cannot run dependencies jobs for security reasons".to_string(), @@ -5117,11 +5110,7 @@ async fn run_dependencies_job( &db, PushIsolationLevel::IsolatedRoot(db.clone()), &w_id, - JobPayload::RawScriptDependencies { - script_path: script_path, - content: raw_code, - language: language, - }, + JobPayload::RawScriptDependencies { script_path, content: raw_code, language }, args, authed.display_username(), &authed.email, @@ -5153,6 +5142,7 @@ async fn run_dependencies_job( pub struct RunFlowDependenciesRequest { pub path: String, pub flow_value: FlowValue, + pub raw_deps: Option>, } #[derive(Serialize)] @@ -5166,21 +5156,27 @@ async fn run_flow_dependencies_job( Path(w_id): Path, Json(req): Json, ) -> error::Result { + check_scopes(&authed, || format!("jobs:run"))?; if authed.is_operator { return Err(error::Error::NotAuthorized( "Operators cannot run dependencies jobs for security reasons".to_string(), )); } + // Create args HashMap with skip_flow_update and raw_deps if present + let mut args_map = HashMap::from([("skip_flow_update".to_string(), to_raw_value(&true))]); + + // Add raw_deps to args if present + if let Some(ref raw_deps) = req.raw_deps { + args_map.insert("raw_deps".to_string(), to_raw_value(raw_deps)); + } + let (uuid, tx) = push( &db, PushIsolationLevel::IsolatedRoot(db.clone()), &w_id, JobPayload::RawFlowDependencies { path: req.path, flow_value: req.flow_value }, - PushArgs::from(&HashMap::from([( - "skip_flow_update".to_string(), - to_raw_value(&true), - )])), + PushArgs::from(&args_map), authed.display_username(), &authed.email, username_to_permissioned_as(&authed.username), @@ -5311,10 +5307,10 @@ async fn add_batch_jobs( } else if let Some(path) = batch_info.path { let mut tx = user_db.clone().begin(&authed).await?; let value_json = sqlx::query!( - "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow + "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - LEFT JOIN flow_version_lite + LEFT JOIN flow_version_lite ON flow_version_lite.id = flow_version.id WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1", &path, &w_id @@ -5428,7 +5424,7 @@ async fn add_batch_jobs( ) INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag) - (SELECT uuid, $1, $2, $3 FROM uuid_table) + (SELECT uuid, $1, $2, $3 FROM uuid_table) RETURNING id"#, w_id, Utc::now(), @@ -5446,7 +5442,7 @@ async fn add_batch_jobs( .await?; sqlx::query!( - "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", &uuids, authed.email, @@ -5473,7 +5469,7 @@ async fn add_batch_jobs( if let Some(custom_concurrency_key) = custom_concurrency_key { sqlx::query!( - "INSERT INTO concurrency_counter(concurrency_id, job_uuids) + "INSERT INTO concurrency_counter(concurrency_id, job_uuids) VALUES ($1, '{}'::jsonb)", &custom_concurrency_key ) @@ -5501,7 +5497,6 @@ async fn run_preview_flow_job( Query(run_query): Query, Json(raw_flow): Json, ) -> error::Result<(StatusCode, String)> { - check_scopes(&authed, || format!("jobs:runflow"))?; if authed.is_operator { return Err(error::Error::NotAuthorized( "Operators cannot run preview jobs for security reasons".to_string(), @@ -5602,7 +5597,8 @@ pub async fn run_job_by_hash_inner( delete_after_use, .. } = get_script_info_for_hash(&mut *tx, &w_id, hash).await?; - check_scopes(&authed, || format!("run:script/{path}"))?; + + check_scopes(&authed, || format!("jobs:run:scripts:{path}"))?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); } @@ -5750,6 +5746,7 @@ async fn get_job_update( Path((w_id, job_id)): Path<(String, Uuid)>, Query(JobUpdateQuery { log_offset, get_progress, running, only_result }): Query, ) -> JsonResult { +<<<<<<< HEAD Ok(Json( get_job_update_data( &opt_authed, @@ -5767,6 +5764,54 @@ async fn get_job_update( .await?, )) } +======= + let record = sqlx::query!( + "SELECT + c.id IS NOT NULL AS completed, + CASE + WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END) + ELSE false + END AS running, + SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs, + COALESCE(r.memory_peak, c.memory_peak) AS mem_peak, + CASE + -- flow step: + WHEN flow_step_id IS NOT NULL THEN NULL + -- completed: + WHEN c.id IS NOT NULL THEN COALESCE( + c.workflow_as_code_status || c.flow_status, + c.workflow_as_code_status, + c.flow_status + ) + -- not completed: + ELSE COALESCE( + f.workflow_as_code_status || f.flow_status, + f.workflow_as_code_status, + f.flow_status + ) + END AS \"flow_status: sqlx::types::Json>\", + job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset, + created_by AS \"created_by!\", + CASE WHEN $4::BOOLEAN THEN ( + SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc' + ) END AS progress + FROM v2_job j + LEFT JOIN v2_job_queue q USING (id) + LEFT JOIN v2_job_runtime r USING (id) + LEFT JOIN v2_job_status f USING (id) + LEFT JOIN v2_job_completed c USING (id) + LEFT JOIN job_logs ON job_logs.job_id = $3 + WHERE j.workspace_id = $2 AND j.id = $3", + log_offset, + &w_id, + job_id, + get_progress.unwrap_or(false), + running, + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?; +>>>>>>> main async fn get_job_update_sse( OptAuthed(opt_authed): OptAuthed, @@ -6276,8 +6321,6 @@ async fn list_completed_jobs( Query(pagination): Query, Query(lq): Query, ) -> error::JsonResult> { - check_scopes(&authed, || format!("jobs:listjobs"))?; - let (per_page, offset) = paginate(pagination); let sql = list_completed_jobs_query( @@ -6615,8 +6658,6 @@ async fn delete_completed_job<'a>( Extension(db): Extension, Path((w_id, id)): Path<(String, Uuid)>, ) -> error::Result { - check_scopes(&authed, || format!("jobs:deletejob"))?; - let mut tx = user_db.begin(&authed).await?; require_admin(authed.is_admin, &authed.username)?; diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 38bd67e675..21d58a7fe6 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -34,11 +34,11 @@ use agent_workers_oss::AgentCache; use anyhow::Context; use argon2::Argon2; -use axum::extract::DefaultBodyLimit; -use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Router}; -use axum::response::Response; -use axum::http::HeaderValue; use axum::body::Body; +use axum::extract::DefaultBodyLimit; +use axum::http::HeaderValue; +use axum::response::Response; +use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Router}; use db::DB; use reqwest::Client; #[cfg(feature = "oauth2")] @@ -153,6 +153,7 @@ mod schedule; #[cfg(feature = "private")] pub mod scim_ee; mod scim_oss; +mod scopes; mod scripts; mod service_logs; mod settings; @@ -178,6 +179,7 @@ mod stripe_oss; #[cfg(feature = "private")] pub mod teams_ee; mod teams_oss; +mod token; mod tracing_init; mod triggers; mod users; @@ -239,7 +241,6 @@ lazy_static::lazy_static! { } - // Compliance with cloud events spec. pub async fn add_webhook_allowed_origin( req: axum::extract::Request, @@ -262,7 +263,6 @@ pub async fn add_webhook_allowed_origin( next.run(req).await } - #[cfg(not(feature = "tantivy"))] type IndexReader = (); @@ -649,6 +649,7 @@ pub async fn run_server( scim_oss::global_service() .route_layer(axum::middleware::from_fn(has_scim_token)), ) + .nest("/tokens", token::global_service()) .nest("/concurrency_groups", concurrency_groups::global_service()) .nest("/scripts_u", scripts::global_unauthed_service()) .nest("/apps_u", { diff --git a/backend/windmill-api/src/mqtt_triggers.rs b/backend/windmill-api/src/mqtt_triggers.rs index 73f40c6307..ae62f662e7 100644 --- a/backend/windmill-api/src/mqtt_triggers.rs +++ b/backend/windmill-api/src/mqtt_triggers.rs @@ -4,6 +4,7 @@ use crate::{ resources::try_get_resource_from_db_as, trigger_helpers::{trigger_runnable, TriggerJobArgs}, users::fetch_api_authed, + utils::check_scopes, }; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; @@ -519,6 +520,9 @@ pub async fn create_mqtt_trigger( Path(w_id): Path, Json(new_mqtt_trigger): Json, ) -> error::Result<(StatusCode, String)> { + check_scopes(&authed, || { + format!("mqtt_triggers:write:{}", &new_mqtt_trigger.path) + })?; if *CLOUD_HOSTED { return Err(error::Error::BadRequest( "MQTT triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(), @@ -701,8 +705,10 @@ pub async fn get_mqtt_trigger( Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); + check_scopes(&authed, || format!("mqtt_triggers:read:{}", path))?; + + let mut tx = user_db.begin(&authed).await?; let trigger = sqlx::query_as!( MqttTrigger, r#" @@ -754,6 +760,8 @@ pub async fn update_mqtt_trigger( Json(mqtt_trigger): Json, ) -> error::Result { let workspace_path = path.to_path(); + check_scopes(&authed, || format!("mqtt_triggers:write:{}", workspace_path))?; + let EditMqttTrigger { mqtt_resource_path, subscribe_topics, @@ -856,6 +864,8 @@ pub async fn delete_mqtt_trigger( Path((w_id, path)): Path<(String, StripPath)>, ) -> error::Result { let path = path.to_path(); + check_scopes(&authed, || format!("mqtt_triggers:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; sqlx::query!( r#" @@ -931,8 +941,9 @@ pub async fn set_enabled( Path((w_id, path)): Path<(String, StripPath)>, Json(payload): Json, ) -> error::Result { - let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); + check_scopes(&authed, || format!("mqtt_triggers:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; // important to set server_id, last_server_ping and error to NULL to stop current mqtt listener let one_o = sqlx::query_scalar!( diff --git a/backend/windmill-api/src/postgres_triggers/handler.rs b/backend/windmill-api/src/postgres_triggers/handler.rs index c7f4b233ed..44318200f4 100644 --- a/backend/windmill-api/src/postgres_triggers/handler.rs +++ b/backend/windmill-api/src/postgres_triggers/handler.rs @@ -6,6 +6,7 @@ use std::collections::{ use crate::{ db::{ApiAuthed, DB}, postgres_triggers::mapper::{Mapper, MappingInfo}, + utils::check_scopes, }; use axum::{ extract::{Path, Query}, @@ -373,6 +374,8 @@ pub async fn create_postgres_trigger( Path(w_id): Path, Json(new_postgres_trigger): Json, ) -> Result<(StatusCode, String)> { + check_scopes(&authed, || format!("postgres_triggers:write:{}", new_postgres_trigger.path))?; + if *CLOUD_HOSTED { return Err(error::Error::BadRequest( "Postgres triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(), @@ -1150,8 +1153,9 @@ pub async fn get_postgres_trigger( Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); + check_scopes(&authed, || format!("postgres_triggers:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; let trigger = sqlx::query_as!( PostgresTrigger, r#" @@ -1200,6 +1204,7 @@ pub async fn update_postgres_trigger( Json(postgres_trigger): Json, ) -> Result { let workspace_path = path.to_path(); + check_scopes(&authed, || format!("postgres_triggers:write:{}", workspace_path))?; let EditPostgresTrigger { replication_slot_name, @@ -1332,6 +1337,7 @@ pub async fn delete_postgres_trigger( Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("postgres_triggers:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; sqlx::query!( r#" @@ -1403,8 +1409,9 @@ pub async fn set_enabled( Path((w_id, path)): Path<(String, StripPath)>, Json(payload): Json, ) -> Result { - let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); + check_scopes(&authed, || format!("postgres_triggers:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; // important to set server_id, last_server_ping and error to NULL to stop current postgres listener let one_o = sqlx::query_scalar!( diff --git a/backend/windmill-api/src/raw_apps.rs b/backend/windmill-api/src/raw_apps.rs index 3174102932..0dd5b8c8b7 100644 --- a/backend/windmill-api/src/raw_apps.rs +++ b/backend/windmill-api/src/raw_apps.rs @@ -8,6 +8,7 @@ use crate::{ db::{ApiAuthed, DB}, users::require_owner_of_path, + utils::check_scopes, webhook_util::{WebhookMessage, WebhookShared}, }; use axum::{ @@ -128,6 +129,7 @@ async fn get_data( Path((w_id, _version, path)): Path<(String, u16, StripPath)>, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("raw_apps:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let app_o = sqlx::query_scalar!( @@ -154,6 +156,7 @@ async fn create_app( Path(w_id): Path, Json(app): Json, ) -> Result<(StatusCode, String)> { + check_scopes(&authed, || format!("raw_apps:write:{}", app.path))?; if *CLOUD_HOSTED { let nb_apps = sqlx::query_scalar!( "SELECT COUNT(*) FROM raw_app WHERE workspace_id = $1", @@ -233,6 +236,7 @@ async fn delete_app( Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("raw_apps:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; sqlx::query!( @@ -271,6 +275,7 @@ async fn update_app( use sql_builder::prelude::*; let path = path.to_path(); + check_scopes(&authed, || format!("raw_apps:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; let mut sqlb = SqlBuilder::update_table("raw_app"); @@ -347,6 +352,7 @@ async fn exists_app( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { let path = path.to_path(); + // Note: exists_app doesn't require authentication, so no scope check needed let exists = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)", path, diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index fd988a271c..77585fb147 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -11,6 +11,7 @@ use std::collections::HashMap; use crate::{ db::{ApiAuthed, DB}, users::{maybe_refresh_folders, require_owner_of_path, Tokened}, + utils::check_scopes, webhook_util::{WebhookMessage, WebhookShared}, }; use axum::{ @@ -280,6 +281,7 @@ async fn get_resource( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("resources:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let resource_o = sqlx::query_as!( @@ -331,6 +333,7 @@ async fn get_resource_value( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult> { let path = path.to_path(); + check_scopes(&authed, || format!("resources:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let value_o = sqlx::query_scalar!( @@ -422,12 +425,15 @@ async fn get_resource_value_interpolated( Path((w_id, path)): Path<(String, StripPath)>, Query(job_info): Query, ) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:read:{}", path))?; + return get_resource_value_interpolated_internal( &authed, Some(user_db), &db, w_id.as_str(), - path.to_path(), + path, job_info.job_id, token.as_str(), ) @@ -658,6 +664,7 @@ async fn create_resource( Query(q): Query, Json(resource): Json, ) -> Result<(StatusCode, String)> { + check_scopes(&authed, || format!("resources:write:{}", resource.path))?; if *CLOUD_HOSTED { let nb_resources = sqlx::query_scalar!( "SELECT COUNT(*) FROM resource WHERE workspace_id = $1", @@ -740,6 +747,7 @@ async fn delete_resource( Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("resources:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; sqlx::query!( @@ -798,6 +806,7 @@ async fn update_resource( use sql_builder::prelude::*; let path = path.to_path(); + check_scopes(&authed, || format!("resources:write:{}", path))?; let mut sqlb = SqlBuilder::update_table("resource"); sqlb.and_where_eq("path", "?".bind(&path)); @@ -891,6 +900,7 @@ async fn update_resource_value( Json(nv): Json, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("resources:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; sqlx::query!( diff --git a/backend/windmill-api/src/schedule.rs b/backend/windmill-api/src/schedule.rs index f5a1aa0fcd..58566f4962 100644 --- a/backend/windmill-api/src/schedule.rs +++ b/backend/windmill-api/src/schedule.rs @@ -7,7 +7,10 @@ */ use crate::{ - db::{ApiAuthed, DB}, settings::{delete_global_setting, set_global_setting_internal}, users::maybe_refresh_folders, utils::require_super_admin + db::{ApiAuthed, DB}, + settings::{delete_global_setting, set_global_setting_internal}, + users::maybe_refresh_folders, + utils::{check_scopes, require_super_admin}, }; use axum::{ extract::{Extension, Path, Query}, @@ -128,6 +131,8 @@ async fn create_schedule( Path(w_id): Path, Json(ns): Json, ) -> Result { + check_scopes(&authed, || format!("schedules:write:{}", ns.path))?; + let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await; #[cfg(not(feature = "enterprise"))] @@ -294,6 +299,7 @@ async fn edit_schedule( Json(es): Json, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("schedules:write:{}", path))?; let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await; let mut tx = user_db.begin(&authed).await?; @@ -560,6 +566,7 @@ async fn get_schedule( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("schedules:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let schedule_o = windmill_queue::schedule::get_schedule_opt(&mut *tx, &w_id, path).await?; @@ -608,6 +615,7 @@ pub async fn set_enabled( ) -> Result { let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); + check_scopes(&authed, || format!("schedules:write:{}", path))?; let schedule_o = sqlx::query_as!( Schedule, r#" @@ -745,8 +753,9 @@ async fn delete_schedule( Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { - let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); + check_scopes(&authed, || format!("schedules:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; clear_schedule(&mut tx, path, &w_id).await?; let exists = sqlx::query_scalar!( diff --git a/backend/windmill-api/src/scopes.rs b/backend/windmill-api/src/scopes.rs new file mode 100644 index 0000000000..d437e8877a --- /dev/null +++ b/backend/windmill-api/src/scopes.rs @@ -0,0 +1,868 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use windmill_common::error::{Error, Result}; + +/// Comprehensive scope system for JWT token authorization +/// +/// Scopes follow the format: {domain}:{action}[:{resource}] +/// Examples: +/// - "jobs:read" - Read access to jobs +/// - "scripts:write:f/folder/*" - Write access to scripts in a folder +/// - "*" - Full access (superuser) + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ScopeDefinition { + pub domain: String, + pub action: String, + pub kind: Option, // For jobs:run:kind (optional) + pub resource: Option>, +} + +impl ScopeDefinition { + pub fn new( + domain: &str, + action: &str, + kind: Option<&str>, + resource: Option>, + ) -> Self { + Self { + domain: domain.to_string(), + action: action.to_string(), + kind: kind.map(|s| s.to_string()), + resource: resource, + } + } + + pub fn from_scope_string(scope: &str) -> Result { + let parts: Vec<&str> = scope.split(':').collect(); + + let into_owned_vec = |resources: &str| -> Vec { + let resources = resources + .split(",") + .collect_vec() + .into_iter() + .map(ToOwned::to_owned) + .collect_vec(); + + resources + }; + + match parts.len() { + 2 => Ok(Self::new(parts[0], parts[1], None, None)), // domain:action + 3 => { + if parts[0] == "jobs" && parts[1] == "run" { + Ok(Self::new(parts[0], parts[1], Some(parts[2]), None)) + } else { + Ok(Self::new( + parts[0], + parts[1], + None, + Some(into_owned_vec(parts[2])), + )) + } + } + 4 => { + if parts[0] == "jobs" && parts[1] == "run" { + Ok(Self::new( + parts[0], + parts[1], + Some(parts[2]), + Some(into_owned_vec(parts[3])), + )) + } else { + Err(Error::BadRequest(format!( + "Invalid 4-part scope: {}", + scope + ))) + } + } + _ => Err(Error::BadRequest(format!( + "Invalid scope format: {}", + scope + ))), + } + } + + pub fn as_string(&self) -> String { + match (&self.kind, &self.resource) { + (Some(kind), Some(resource)) => { + format!( + "{}:{}:{}:{}", + self.domain, + self.action, + kind, + resource.join(",") + ) + } + (Some(kind), None) => { + format!("{}:{}:{}", self.domain, self.action, kind) + } + (None, Some(resource)) => { + format!("{}:{}:{}", self.domain, self.action, resource.join(",")) + } + (None, None) => format!("{}:{}", self.domain, self.action), + } + } + + pub fn includes(&self, other: &ScopeDefinition) -> bool { + if self.domain != other.domain { + return false; + } + + match (self.action.as_str(), other.action.as_str()) { + (a, b) if (a == "write" && b == "read") || (a == b) => {} + _ => return false, + } + + if self.domain == "jobs" && self.action == "run" { + match (&self.kind, &other.kind) { + (Some(self_kind), Some(other_kind)) => { + if self_kind != other_kind { + return false; + } + } + (Some(_), None) => { + return false; + } + (None, _) => { + return true; + } + } + } + + match (&self.resource, &other.resource) { + (Some(self_resources), Some(other_resources)) => { + resources_match(self_resources, other_resources) + } + (Some(_), None) => false, + (None, _) => true, + } + } +} + +fn resources_match(scope_resources: &[String], accepted_resources: &[String]) -> bool { + if scope_resources.contains(&"*".to_string()) || accepted_resources.contains(&"*".to_string()) { + return true; + } + + if scope_resources.len() <= 4 && accepted_resources.len() <= 4 { + return resources_match_small(scope_resources, accepted_resources); + } + + resources_match_large(scope_resources, accepted_resources) +} + +fn resources_match_small(scope_resources: &[String], accepted_resources: &[String]) -> bool { + for required in accepted_resources { + for scope_resource in scope_resources { + if resource_matches_pattern(scope_resource, required) { + return true; + } + } + } + false +} + +fn resources_match_large(scope_resources: &[String], accepted_resources: &[String]) -> bool { + let mut exact_matches = HashSet::new(); + let mut patterns = Vec::new(); + + for scope_resource in scope_resources { + if scope_resource.contains('*') { + patterns.push(scope_resource); + } else { + exact_matches.insert(scope_resource); + } + } + + for accepted_resource in accepted_resources { + if exact_matches.contains(accepted_resource) { + return true; + } + + for pattern in &patterns { + if resource_matches_pattern(pattern, accepted_resource) { + return true; + } + } + } + + false +} + +fn resource_matches_pattern(scope_resource: &str, accepted_resource: &str) -> bool { + if scope_resource == accepted_resource { + return true; + } + + let matches_wildcard = |pattern: &str, resource: &str| -> bool { + if !pattern.ends_with("/*") { + return false; + } + + let prefix = &pattern[..pattern.len() - 2]; + + if !resource.starts_with(prefix) { + return false; + } + + // If the resource is exactly the prefix, it matches + if resource.len() == prefix.len() { + return true; + } + + // If the resource is longer, the next character must be '/' for a valid match + // This prevents "u/user" from matching "u/use/*" + resource.chars().nth(prefix.len()) == Some('/') + }; + + // Check if either resource is a wildcard pattern and matches the other + matches_wildcard(scope_resource, accepted_resource) + || matches_wildcard(accepted_resource, scope_resource) +} + +/// Available scope domains (top-level API categories) +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ScopeDomain { + // Core resource domains + Jobs, + Scripts, + Flows, + Apps, + Variables, + Resources, + Schedules, + Folders, + Users, + Groups, + Workspaces, + + // Trigger domains + HttpTriggers, + WebsocketTriggers, + KafkaTriggers, + NatsTriggers, + MqttTriggers, + SqsTriggers, + GcpTriggers, + PostgresTriggers, + + // System domains + Audit, + Settings, + Workers, + ServiceLogs, + Configs, + OAuth, + AI, + + Indexer, + Teams, // Microsoft Teams integration + GitSync, // Git synchronization + + // Special domains + Capture, // Webhook capture + Drafts, // Draft resources + Favorites, // User favorites + Inputs, // Input templates + JobHelpers, // Job helper functions + ConcurrencyGroups, // Concurrency groups + Oidc, // OpenID Connect + Openapi, // OpenAPI generation + + // Additional domains + Acls, // Granular access control lists + RawApps, // Raw application data + AgentWorkers, // Agent workers management +} + +impl ScopeDomain { + pub fn as_str(&self) -> &'static str { + match self { + Self::Jobs => "jobs", + Self::Scripts => "scripts", + Self::Flows => "flows", + Self::Apps => "apps", + Self::Variables => "variables", + Self::Resources => "resources", + Self::Schedules => "schedules", + Self::Folders => "folders", + Self::Users => "users", + Self::Groups => "groups", + Self::Workspaces => "workspaces", + Self::HttpTriggers => "http_triggers", + Self::WebsocketTriggers => "websocket_triggers", + Self::KafkaTriggers => "kafka_triggers", + Self::NatsTriggers => "nats_triggers", + Self::MqttTriggers => "mqtt_triggers", + Self::SqsTriggers => "sqs_triggers", + Self::GcpTriggers => "gcp_triggers", + Self::PostgresTriggers => "postgres_triggers", + Self::Audit => "audit", + Self::Settings => "settings", + Self::Workers => "workers", + Self::ServiceLogs => "service_logs", + Self::Configs => "configs", + Self::OAuth => "oauth", + Self::AI => "ai", + Self::Capture => "capture", + Self::Drafts => "drafts", + Self::Favorites => "favorites", + Self::Inputs => "inputs", + Self::JobHelpers => "job_helpers", + Self::ConcurrencyGroups => "concurrency_groups", + Self::Oidc => "oidc", + Self::Openapi => "openapi", + Self::Acls => "acls", + Self::RawApps => "raw_apps", + Self::AgentWorkers => "agent_workers", + Self::Indexer => "indexer", + Self::Teams => "teams", + Self::GitSync => "git_sync", + } + } + + pub fn from_str(s: &str) -> Option { + match s { + "jobs" | "jobs_u" => Some(Self::Jobs), + "scripts" => Some(Self::Scripts), + "flows" => Some(Self::Flows), + "apps" => Some(Self::Apps), + "variables" => Some(Self::Variables), + "resources" => Some(Self::Resources), + "schedules" => Some(Self::Schedules), + "folders" => Some(Self::Folders), + "users" => Some(Self::Users), + "groups" => Some(Self::Groups), + "workspaces" => Some(Self::Workspaces), + "http_triggers" => Some(Self::HttpTriggers), + "websocket_triggers" => Some(Self::WebsocketTriggers), + "kafka_triggers" => Some(Self::KafkaTriggers), + "nats_triggers" => Some(Self::NatsTriggers), + "mqtt_triggers" => Some(Self::MqttTriggers), + "sqs_triggers" => Some(Self::SqsTriggers), + "gcp_triggers" => Some(Self::GcpTriggers), + "postgres_triggers" => Some(Self::PostgresTriggers), + "audit" => Some(Self::Audit), + "settings" => Some(Self::Settings), + "workers" => Some(Self::Workers), + "service_logs" => Some(Self::ServiceLogs), + "configs" => Some(Self::Configs), + "oauth" => Some(Self::OAuth), + "ai" => Some(Self::AI), + "indexer" | "srch" => Some(Self::Indexer), + "teams" => Some(Self::Teams), + "git_sync" | "github_app" => Some(Self::GitSync), + "capture" => Some(Self::Capture), + "drafts" => Some(Self::Drafts), + "favorites" => Some(Self::Favorites), + "inputs" => Some(Self::Inputs), + "job_helpers" => Some(Self::JobHelpers), + "concurrency_groups" => Some(Self::ConcurrencyGroups), + "oidc" => Some(Self::Oidc), + "openapi" => Some(Self::Openapi), + "acls" => Some(Self::Acls), + "raw_apps" => Some(Self::RawApps), + "agent_workers" => Some(Self::AgentWorkers), + _ => None, + } + } +} + +/// Available scope actions +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ScopeAction { + Read, // GET operations, list, view + Write, // POST, PUT, PATCH, DELETE operations, create, update, delete + Run, // Special action for running (scripts, flows, etc.) +} + +impl ScopeAction { + pub fn as_str(&self) -> &'static str { + match self { + Self::Read => "read", + Self::Write => "write", + Self::Run => "run", + } + } + + pub fn from_str(s: &str) -> Option { + match s { + "read" => Some(Self::Read), + "write" => Some(Self::Write), + "delete" => Some(Self::Write), + "run" => Some(Self::Run), + _ => None, + } + } + + /// Check if this action includes another action + /// Write includes Read + pub fn includes(&self, other: &ScopeAction) -> bool { + match (self, other) { + (ScopeAction::Write, ScopeAction::Read) => true, + (a, b) => a == b, + } + } +} + +pub fn check_route_access( + token_scopes: &[String], + route_path: &str, + http_method: &str, +) -> Result<()> { + // Map HTTP method to scope action (considering route context) + let required_action = map_http_method_to_action(http_method, route_path); + + // Find the domain and kind for this route + let (required_domain, required_kind) = extract_domain_from_route(route_path)?; + let mut is_scoped_token = false; + // Check if any token scope grants the required access + for scope_str in token_scopes { + if !scope_str.starts_with("if_jobs:filter_tags:") { + if let Ok(scope) = ScopeDefinition::from_scope_string(scope_str) { + if scope_grants_access( + &scope, + required_domain, + required_action, + required_kind.as_deref(), + )? { + return Ok(()); + } + } + if !is_scoped_token { + is_scoped_token = true; + } + } + } + + //Edge case for backward compatibility, if only scopes defined was filter tag then don't treat this we don't treat the token + //as a restricted token + if !is_scoped_token { + return Ok(()); + } + let scope_display = if let Some(kind) = required_kind { + format!( + "{}:{}:{}", + required_domain.as_str(), + required_action.as_str(), + kind + ) + } else { + format!("{}:{}", required_domain.as_str(), required_action.as_str()) + }; + + Err(Error::NotAuthorized(format!( + "Access denied. Required scope: {}", + scope_display + ))) +} + +const SCRIPT_JOBS: [&'static str; 6] = [ + "jobs/run/p", + "jobs/run/h", + "jobs/run_wait_result/p", + "jobs/run_wait_result/h", + "jobs/run/preview_bundle", + "jobs/run/preview", +]; + +const FLOW_JOBS: [&'static str; 5] = [ + "jobs/run/f", + "jobs/run_wait_result/f", + "jobs/run/preview_flow", + "jobs/restart/f", + "jobs/flow/resume", +]; + +lazy_static::lazy_static! { + static ref RUN_PATH_ACTIONS: Vec<&'static str> = { + let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies",]; + + v.extend(SCRIPT_JOBS); + v.extend(FLOW_JOBS); + v + }; +} + +fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction { + if RUN_PATH_ACTIONS + .iter() + .any(|run_path| route_path.contains(run_path)) + { + return ScopeAction::Run; + } + + match method.to_uppercase().as_str() { + "GET" | "HEAD" | "OPTIONS" => ScopeAction::Read, + "POST" | "PUT" | "PATCH" | "DELETE" => ScopeAction::Write, + _ => ScopeAction::Read, + } +} + +/// Checks the route path to determine the runnable kind (either "flows" or "scripts"). +/// +/// The order of checks is important: +/// - Flow-related paths are checked first to avoid false positives, as some flow paths +/// (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`). +/// +/// Returns `"flows"` or `"scripts"` based on the match, or `None` if no match is found. +fn determine_kind_from_route(route_path: &str) -> Option { + if route_path.starts_with("jobs") { + if FLOW_JOBS.iter().any(|path| route_path.starts_with(path)) { + return Some("flows".to_string()); + } else if SCRIPT_JOBS.iter().any(|path| route_path.starts_with(path)) { + return Some("scripts".to_string()); + } + } + None +} + +fn extract_domain_from_route(route_path: &str) -> Result<(ScopeDomain, Option)> { + // Examples: + // - /api/w/workspace/jobs/123 -> jobs domain (workspaced) + // - /api/teams/sync -> teams domain (global) + // - /api/srch/index/search -> indexer domain (global) + let parts: Vec<&str> = route_path.split('/').collect(); + + let (domain, kind) = if parts.len() >= 5 && parts[1] == "api" && parts[2] == "w" { + let domain_part = parts[4]; + let route_suffix = &parts[4..].join("/"); + + let domain = ScopeDomain::from_str(domain_part); + + let kind = determine_kind_from_route(route_suffix); + + (domain, kind) + } else if parts.len() >= 3 && parts[1] == "api" { + (ScopeDomain::from_str(parts[2]), None) + } else { + (None, None) + }; + + if let Some(domain) = domain { + return Ok((domain, kind)); + } + + Err(Error::BadRequest(format!( + "Could not extract domain from route: {}", + route_path + ))) +} + +fn scope_grants_access( + scope: &ScopeDefinition, + required_domain: ScopeDomain, + required_action: ScopeAction, + required_kind: Option<&str>, +) -> Result { + // Check domain match + let scope_domain = ScopeDomain::from_str(&scope.domain) + .ok_or_else(|| Error::BadRequest(format!("Invalid scope domain: {}", scope.domain)))?; + + if scope_domain != required_domain { + return Ok(false); + } + + // Check action match (with hierarchical permissions) + let scope_action = ScopeAction::from_str(&scope.action) + .ok_or_else(|| Error::BadRequest(format!("Invalid scope action: {}", scope.action)))?; + + if !scope_action.includes(&required_action) { + return Ok(false); + } + + if required_domain == ScopeDomain::Jobs && required_action == ScopeAction::Run { + match (&scope.kind, required_kind) { + (Some(scope_kind), Some(req_kind)) => { + if scope_kind != req_kind { + return Ok(false); + } + } + (None, _) => {} + (Some(_), None) => { + return Ok(false); + } + } + } + + // No resource specified means access to entire domain + Ok(true) +} + +/// Helper function to check if scopes allow access to a route +pub fn check_scopes_for_route( + token_scopes: Option<&[String]>, + route_path: &str, + http_method: &str, +) -> Result<()> { + // If no scopes defined, allow access (backward compatibility) + let scopes = match token_scopes { + Some(s) if !s.is_empty() => s, + _ => return Ok(()), + }; + + check_route_access(scopes, route_path, http_method) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_scope_definition_parsing() { + let scope = ScopeDefinition::from_scope_string("jobs:read").unwrap(); + assert_eq!(scope.domain, "jobs"); + assert_eq!(scope.action, "read"); + assert_eq!(scope.kind, None); + assert_eq!(scope.resource, None); + + let scope = ScopeDefinition::from_scope_string("jobs:run:scripts:f/folder/*").unwrap(); + assert_eq!(scope.domain, "jobs"); + assert_eq!(scope.action, "run"); + assert_eq!(scope.kind, Some("scripts".to_string())); + assert_eq!(scope.resource, Some(vec!["f/folder/*".to_string()])); + + // Test jobs:run:kind parsing + let scope = ScopeDefinition::from_scope_string("jobs:run:scripts").unwrap(); + assert_eq!(scope.domain, "jobs"); + assert_eq!(scope.action, "run"); + assert_eq!(scope.kind, Some("scripts".to_string())); + assert_eq!(scope.resource, None); + + // Test jobs:run:kind:resource parsing + let scope = ScopeDefinition::from_scope_string("jobs:run:flows:f/folder/*").unwrap(); + assert_eq!(scope.domain, "jobs"); + assert_eq!(scope.action, "run"); + assert_eq!(scope.kind, Some("flows".to_string())); + assert_eq!(scope.resource, Some(vec!["f/folder/*".to_string()])); + + // Test comma-separated resources parsing + let scope = + ScopeDefinition::from_scope_string("scripts:read:path1,path2,f/folder/*").unwrap(); + assert_eq!(scope.domain, "scripts"); + assert_eq!(scope.action, "read"); + assert_eq!(scope.kind, None); + assert_eq!( + scope.resource, + Some(vec![ + "path1".to_string(), + "path2".to_string(), + "f/folder/*".to_string() + ]) + ); + } + + #[test] + fn test_scope_action_hierarchy() { + assert!(ScopeAction::Write.includes(&ScopeAction::Read)); + assert!(!ScopeAction::Read.includes(&ScopeAction::Write)); + assert!(!ScopeAction::Run.includes(&ScopeAction::Read)); + assert!(!ScopeAction::Run.includes(&ScopeAction::Write)); + } + + #[test] + fn test_route_domain_extraction() { + let (domain, kind) = extract_domain_from_route("/api/w/test_workspace/jobs/123").unwrap(); + assert_eq!(domain, ScopeDomain::Jobs); + assert_eq!(kind, None); + + let (domain, kind) = + extract_domain_from_route("/api/w/test_workspace/scripts/test_script").unwrap(); + assert_eq!(domain, ScopeDomain::Scripts); + assert_eq!(kind, None); + } + + #[test] + fn test_specific_scope_access() { + let scopes = vec!["jobs:read".to_string()]; + + assert!(check_route_access(&scopes, "/api/w/test_workspace/jobs/123", "GET").is_ok()); + + // DELETE now requires write permission, so it should still fail with read-only scope + assert!(check_route_access(&scopes, "/api/w/test_workspace/jobs/123", "DELETE").is_err()); + } + + #[test] + fn test_new_domain_parsing() { + // Test that new domains are properly parsed + assert_eq!(ScopeDomain::from_str("acls"), Some(ScopeDomain::Acls)); + assert_eq!( + ScopeDomain::from_str("raw_apps"), + Some(ScopeDomain::RawApps) + ); + assert_eq!( + ScopeDomain::from_str("agent_workers"), + Some(ScopeDomain::AgentWorkers) + ); + + // Test that string conversion works both ways + assert_eq!(ScopeDomain::Acls.as_str(), "acls"); + assert_eq!(ScopeDomain::RawApps.as_str(), "raw_apps"); + assert_eq!(ScopeDomain::AgentWorkers.as_str(), "agent_workers"); + } + + #[test] + fn test_resource_array_matching() { + // Test wildcard access + let scope_all = ScopeDefinition::new("scripts", "read", None, Some(vec!["*".to_string()])); + let required = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["path1".to_string(), "path2".to_string()]), + ); + assert!(scope_all.includes(&required)); + + // Test exact matches + let scope_exact = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["path1".to_string(), "path2".to_string()]), + ); + let required_subset = + ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()])); + assert!(scope_exact.includes(&required_subset)); + + // Test partial match - should grant access if ANY required resource matches + let scope_limited = + ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()])); + let required_partial = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["path1".to_string(), "path2".to_string()]), + ); + assert!(scope_limited.includes(&required_partial)); // path1 matches, so access granted + + // Test no match - scope doesn't cover any of the required resources + let scope_different = + ScopeDefinition::new("scripts", "read", None, Some(vec!["path3".to_string()])); + let required_no_match = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["path1".to_string(), "path2".to_string()]), + ); + assert!(!scope_different.includes(&required_no_match)); + + // Test pattern matching + let scope_pattern = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["f/folder/*".to_string()]), + ); + let required_in_folder = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["f/folder/script1".to_string()]), + ); + assert!(scope_pattern.includes(&required_in_folder)); + + // Test mixed patterns and exact matches + let scope_mixed = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["exact_path".to_string(), "f/folder/*".to_string()]), + ); + let required_mixed1 = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["exact_path".to_string()]), + ); + let required_mixed2 = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["f/folder/script2".to_string()]), + ); + assert!(scope_mixed.includes(&required_mixed1)); + assert!(scope_mixed.includes(&required_mixed2)); + } + + #[test] + fn test_efficiency_small_vs_large_arrays() { + // Test small array optimization path + let scope_small = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["path1".to_string(), "path2".to_string()]), + ); + let required_small = + ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()])); + assert!(scope_small.includes(&required_small)); + + // Test large array optimization path + let large_scope_vec: Vec = (0..10).map(|i| format!("path{}", i)).collect(); + let scope_large = ScopeDefinition::new("scripts", "read", None, Some(large_scope_vec)); + let required_large = + ScopeDefinition::new("scripts", "read", None, Some(vec!["path5".to_string()])); + assert!(scope_large.includes(&required_large)); + } + + #[test] + fn test_user_example_case() { + // User's example: scope has "u/dieri/*", required has ["u/dadad/wqdq", "u/*"] + // Should grant access because scope "u/dieri/*" falls under required pattern "u/*" + let user_scope = + ScopeDefinition::new("scripts", "read", None, Some(vec!["u/dieri/*".to_string()])); + let required_mixed = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["u/dadad/wqdq".to_string(), "u/*".to_string()]), + ); + assert!(user_scope.includes(&required_mixed)); // Should match because u/dieri/* falls under u/* + + // Another example: scope covers one but not both paths + let scope_specific = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["folder/file1".to_string()]), + ); + let required_multi = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["folder/file1".to_string(), "other/file2".to_string()]), + ); + assert!(scope_specific.includes(&required_multi)); // Should match because folder/file1 matches exactly + + // Test bidirectional pattern matching more explicitly + let scope_broad = + ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()])); + let required_specific = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["u/dieri/script.py".to_string()]), + ); + assert!(scope_broad.includes(&required_specific)); // u/* covers u/dieri/script.py + + let scope_specific_path = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["u/dieri/script.py".to_string()]), + ); + let required_broad = + ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()])); + assert!(scope_specific_path.includes(&required_broad)); // u/dieri/script.py satisfies u/* + } +} diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 4887f9e6ac..e32a2da82b 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -14,7 +14,7 @@ use crate::{ get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail, }, users::{maybe_refresh_folders, require_owner_of_path}, - utils::WithStarredInfoQuery, + utils::{check_scopes, WithStarredInfoQuery}, webhook_util::{WebhookMessage, WebhookShared}, HTTP_CLIENT, }; @@ -43,7 +43,7 @@ use windmill_audit::ActionKind; use windmill_worker::process_relative_imports; use windmill_common::{ - assets::{clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind}, + assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType}, error::to_anyhow, worker::CLOUD_HOSTED, }; @@ -116,6 +116,9 @@ pub struct ScriptWDraft { pub has_preprocessor: Option, #[serde(skip_serializing_if = "Option::is_none")] pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[sqlx(json(nullable))] + pub assets: Option>, } pub fn global_service() -> Router { @@ -516,6 +519,8 @@ async fn create_script_internal<'c>( user_db: UserDB, webhook: WebhookShared, ) -> Result<(ScriptHash, Transaction<'c, Postgres>)> { + check_scopes(&authed, || format!("scripts:write:{}", ns.path))?; + let codebase = ns.codebase.as_ref(); #[cfg(not(feature = "enterprise"))] if ns.ws_error_handler_muted.is_some_and(|val| val) { @@ -769,8 +774,8 @@ async fn create_script_internal<'c>( content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33)", + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)", &w_id, &hash.0, ns.path, @@ -808,6 +813,7 @@ async fn create_script_internal<'c>( None }, validate_schema, + ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok()) ) .execute(&mut *tx) .await?; @@ -922,16 +928,8 @@ async fn create_script_internal<'c>( } clear_asset_usage(&mut *tx, &w_id, &script_path, AssetUsageKind::Script).await?; - for asset in parse_assets(&ns.content, ns.language)?.iter().flatten() { - insert_asset_usage( - &mut *tx, - &w_id, - asset, - ns.fallback_access_types.as_ref().map(Vec::as_slice), - &ns.path, - AssetUsageKind::Script, - ) - .await?; + for asset in ns.assets.as_ref().into_iter().flatten() { + insert_asset_usage(&mut *tx, &w_id, &asset, &ns.path, AssetUsageKind::Script).await?; } let permissioned_as = username_to_permissioned_as(&authed.username); @@ -1062,6 +1060,7 @@ async fn get_script_by_path( Query(query): Query, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("scripts:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let script_o = if query.with_starred_info.unwrap_or(false) { @@ -1119,10 +1118,11 @@ async fn get_script_by_path_w_draft( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("scripts:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 ORDER BY script.created_at DESC LIMIT 1", @@ -1142,6 +1142,8 @@ async fn get_script_history( Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("scripts:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let query_result = sqlx::query!( "SELECT s.hash as hash, dm.deployment_msg as deployment_msg @@ -1149,7 +1151,7 @@ async fn get_script_history( WHERE s.workspace_id = $1 AND s.path = $2 ORDER by s.created_at DESC", w_id, - path.to_path(), + path, ) .fetch_all(&mut *tx) .await?; @@ -1170,6 +1172,8 @@ async fn get_latest_version( Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("scripts:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let row_o = sqlx::query!( "SELECT s.hash as hash, dm.deployment_msg as deployment_msg @@ -1177,7 +1181,7 @@ async fn get_latest_version( WHERE s.workspace_id = $1 AND s.path = $2 ORDER by s.created_at DESC LIMIT 1", w_id, - path.to_path(), + path, ) .fetch_optional(&mut *tx) .await?; @@ -1200,11 +1204,14 @@ async fn update_script_history( Path((w_id, script_hash, script_path)): Path<(String, ScriptHash, StripPath)>, Json(script_history_update): Json, ) -> Result<()> { + let script_path = script_path.to_path(); + check_scopes(&authed, || format!("scripts:write:{}", script_path))?; + let mut tx = user_db.begin(&authed).await?; sqlx::query!( "INSERT INTO deployment_metadata (workspace_id, path, script_hash, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET deployment_msg = $4", w_id, - script_path.to_path(), + script_path, script_hash.0, script_history_update.deployment_msg, ) @@ -1353,6 +1360,7 @@ async fn raw_script_by_path_internal( unpin: bool, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("scripts:read:{}", path))?; if !path.ends_with(".py") && !path.ends_with(".ts") && !path.ends_with(".go") @@ -1502,6 +1510,9 @@ async fn get_script_by_hash( }), ) .await?; + + check_scopes(&authed, || format!("scripts:read:{}", &r.script.path))?; + tx.commit().await?; Ok(Json(r)) @@ -1566,6 +1577,7 @@ async fn archive_script_by_path( Path((w_id, path)): Path<(String, StripPath)>, ) -> Result<()> { let path = path.to_path(); + check_scopes(&authed, || format!("scripts:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; require_owner_of_path(&authed, path)?; @@ -1638,6 +1650,7 @@ async fn archive_script_by_hash( .await .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; + check_scopes(&authed, || format!("scripts:write:{}", &script.path))?; sqlx::query!( "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)", &w_id, @@ -1686,6 +1699,7 @@ async fn delete_script_by_hash( .await .map_err(|e| Error::internal_err(format!("deleting script by hash {w_id}: {e:#}")))?; + check_scopes(&authed, || format!("scripts:write:{}", &script.path))?; sqlx::query!( "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)", &w_id, @@ -1728,6 +1742,7 @@ async fn delete_script_by_path( Query(query): Query, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("scripts:write:{}", path))?; if path == "u/admin/hub_sync" && w_id == "admins" { return Err(Error::BadRequest( diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs new file mode 100644 index 0000000000..9ed17dfc6a --- /dev/null +++ b/backend/windmill-api/src/token.rs @@ -0,0 +1,195 @@ +use axum::{routing::get, Json, Router}; +use lazy_static::lazy_static; +use serde::{Deserialize, Serialize}; +use windmill_common::error::JsonResult; + +#[derive(Default, Serialize, Deserialize, Clone)] +pub struct ScopeOption { + pub value: String, + pub label: String, + pub requires_resource_path: bool, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct ScopeDomain { + pub name: String, + pub description: Option, + pub scopes: Vec, +} + +fn build_trigger_scope_domains() -> Vec { + const TRIGGER_DOMAINS: &[(&str, &str)] = &[ + ("http_triggers", "HTTP"), + ("websocket_triggers", "WebSocket"), + ("kafka_triggers", "Kafka"), + ("nats_triggers", "NATS"), + ("mqtt_triggers", "MQTT"), + ("sqs_triggers", "AWS SQS"), + ("gcp_triggers", "GCP Pub/Sub"), + ("postgres_triggers", "PostgreSQL"), + ]; + + TRIGGER_DOMAINS + .iter() + .map(|(domain, display_name)| ScopeDomain { + name: format!("{} Triggers", display_name), + description: Some(format!("{} trigger management", display_name)), + scopes: vec![ + ScopeOption { + value: format!("{domain}:read"), + label: "Read".to_string(), + requires_resource_path: true, + }, + ScopeOption { + value: format!("{domain}:write"), + label: "Write".to_string(), + requires_resource_path: true, + }, + ], + }) + .collect() +} + +fn build_standard_scope_domains() -> Vec { + const STANDARD_DOMAINS: &[(&str, &str, &str, bool)] = &[ + ( + "scripts", + "Scripts", + "Access to automation scripts and workflows", + true, + ), + ( + "flows", + "Flows", + "Access to automation scripts and workflows", + true, + ), + ("apps", "Apps", "App management", true), + ("raw_apps", "RawApps", "Raw app management", true), + ("resources", "Resources", "Resource management", true), + ("variables", "Variables", "", true), + ( + "schedules", + "Schedules", + "Scheduled tasks and automated triggers", + true, + ), + ("folders", "Folders", "Folder management", true), + ("users", "Users", "User account management", false), + ("groups", "Groups", "Group management", false), + ("workspaces", "Workspaces", "Workspace management", false), + ("audit", "Audit", "Audit log management", false), + ("workers", "Workers", "Worker management", false), + ("settings", "Settings", "System settings management", false), + ( + "service_logs", + "Service Logs", + "Service log management", + false, + ), + ("configs", "Configs", "Configuration management", false), + ("oauth", "OAuth", "OAuth management", false), + ("ai", "AI", "AI feature management", false), + ( + "agent_workers", + "Agent Workers", + "Agent worker management", + false, + ), + ("drafts", "Drafts", "Draft management", false), + ("favorites", "Favorites", "Favorite items management", false), + ("inputs", "Inputs", "Input management", false), + ("job_helpers", "Job Helpers", "Job helper utilities", false), + ( + "openapi", + "OpenAPI", + "OpenAPI documentation management", + false, + ), + ("capture", "Capture", "Request capture management", false), + ( + "concurrency_groups", + "Concurrency Groups", + "Concurrency group management", + false, + ), + ("oidc", "OIDC", "OIDC management", false), + ("acls", "ACLs", "Access Control List management", false), + ("indexer", "Indexer", "Search indexer management", false), + ("teams", "Teams", "Team management", false), + ( + "git_sync", + "Git Sync", + "Git synchronization management", + false, + ), + ]; + + STANDARD_DOMAINS + .iter() + .map(|(key, name, desc, req)| ScopeDomain { + name: name.to_string(), + description: if desc.is_empty() { + None + } else { + Some(desc.to_string()) + }, + scopes: vec![ + ScopeOption { + value: format!("{key}:read"), + label: "Read".to_string(), + requires_resource_path: *req, + }, + ScopeOption { + value: format!("{key}:write"), + label: "Write".to_string(), + requires_resource_path: *req, + }, + ], + }) + .collect() +} + +lazy_static! { + static ref ALL_SCOPES: Vec = { + let mut groups = vec![ScopeDomain { + name: "Jobs".to_string(), + description: Some("Job management".to_string()), + scopes: vec![ + ScopeOption { + value: "jobs:read".to_string(), + label: "Read".to_string(), + requires_resource_path: false, + }, + ScopeOption { + value: "jobs:write".to_string(), + label: "Write".to_string(), + requires_resource_path: false, + }, + ScopeOption { + value: "jobs:run:scripts".to_string(), + label: "Run scripts".to_string(), + requires_resource_path: true, + }, + ScopeOption { + value: "jobs:run:flows".to_string(), + label: "Run flows".to_string(), + requires_resource_path: true, + }, + ], + }]; + + groups.extend(build_standard_scope_domains()); + groups.extend(build_trigger_scope_domains()); + + groups + }; +} + +pub fn global_service() -> Router { + Router::new().route("/list/scopes", get(get_all_available_scopes)) +} + +async fn get_all_available_scopes() -> JsonResult> { + Ok(Json(ALL_SCOPES.clone())) +} diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 770e6c793a..4692eea71a 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -153,23 +153,6 @@ pub async fn maybe_refresh_folders( } } -pub fn check_scopes(authed: &ApiAuthed, required: F) -> error::Result<()> -where - F: FnOnce() -> String, -{ - if authed.scopes.as_ref().is_some_and(|scopes| { - scopes - .iter() - .any(|s| s.starts_with("jobs:") || s.starts_with("run:")) - }) { - let req = &required(); - if !authed.scopes.as_ref().unwrap().contains(req) { - return Err(Error::BadRequest(format!("missing required scope: {req}"))); - } - } - Ok(()) -} - pub fn get_scope_tags(authed: &ApiAuthed) -> Option> { authed.scopes.as_ref()?.iter().find_map(|s| { if s.starts_with("if_jobs:filter_tags:") { diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index b81d4e4bc4..df77e778f2 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -8,7 +8,7 @@ use axum::{body::Body, response::Response}; use regex::Regex; -use serde::{Deserialize, Deserializer}; +use serde::{Deserialize, Deserializer, Serialize}; use sqlx::{Postgres, Transaction}; #[cfg(feature = "enterprise")] use windmill_common::worker::CLOUD_HOSTED; @@ -18,6 +18,8 @@ use windmill_common::{ DB, }; +use crate::{db::ApiAuthed, scopes::ScopeDefinition}; + #[cfg(feature = "enterprise")] use windmill_common::error::JsonResult; @@ -41,6 +43,36 @@ pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> { } } +pub fn check_scopes(authed: &ApiAuthed, required: F) -> error::Result<()> +where + F: FnOnce() -> String, +{ + if let Some(scopes) = authed.scopes.as_ref() { + let mut is_scoped_token = false; + let required_scope = ScopeDefinition::from_scope_string(&required())?; + for scope in scopes { + if !scope.starts_with("if_jobs:filter_tags:") { + if !is_scoped_token { + is_scoped_token = true; + } + + match ScopeDefinition::from_scope_string(scope) { + Ok(scope) if scope.includes(&required_scope) => return Ok(()), + _ => {} + } + } + } + + if is_scoped_token { + return Err(Error::NotAuthorized(format!( + "Required scope: {}", + required_scope.as_string() + ))); + } + } + Ok(()) +} + pub async fn require_devops_role(db: &DB, email: &str) -> error::Result<()> { let is_devops = is_devops_email(db, email).await?; @@ -192,8 +224,6 @@ where Ok(o.filter(|s| !s.trim().is_empty())) } -use serde::Serialize; - #[derive(Serialize)] pub struct CriticalAlert { id: i32, diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index 73aba91229..919396b7ec 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -7,7 +7,10 @@ */ use crate::{ - db::{ApiAuthed, DB}, users::{maybe_refresh_folders, require_owner_of_path}, webhook_util::{WebhookMessage, WebhookShared} + db::{ApiAuthed, DB}, + users::{maybe_refresh_folders, require_owner_of_path}, + utils::check_scopes, + webhook_util::{WebhookMessage, WebhookShared}, }; use axum::{ @@ -135,6 +138,7 @@ async fn get_variable( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("variables:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let variable_o = sqlx::query_as::<_, ListableVariable>( @@ -219,6 +223,7 @@ async fn get_value( Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult { let path = path.to_path(); + check_scopes(&authed, || format!("variables:read:{}", path))?; let tx = user_db.begin(&authed).await?; return get_value_internal(tx, &db, &w_id, &path, &authed) .await @@ -309,6 +314,7 @@ async fn create_variable( Query(AlreadyEncrypted { already_encrypted }): Query, Json(variable): Json, ) -> Result<(StatusCode, String)> { + check_scopes(&authed, || format!("variables:write:{}", variable.path))?; if *CLOUD_HOSTED { let nb_variables = sqlx::query_scalar!( "SELECT COUNT(*) FROM variable WHERE workspace_id = $1", @@ -405,6 +411,7 @@ async fn delete_variable( Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { let path = path.to_path(); + check_scopes(&authed, || format!("variables:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; sqlx::query!( @@ -479,6 +486,7 @@ async fn update_variable( use sql_builder::prelude::*; let path = path.to_path(); + check_scopes(&authed, || format!("variables:write:{}", path))?; let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await; let mut sqlb = SqlBuilder::update_table("variable"); diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs index ead48f8d98..87ecb87e09 100644 --- a/backend/windmill-api/src/websocket_triggers.rs +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -36,6 +36,7 @@ use crate::{ db::{ApiAuthed, DB}, trigger_helpers::{trigger_runnable, trigger_runnable_and_wait_for_raw_result, TriggerJobArgs}, users::fetch_api_authed, + utils::check_scopes, }; use std::borrow::Cow; @@ -183,8 +184,9 @@ async fn get_websocket_trigger( Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> error::JsonResult { - let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); + check_scopes(&authed, || format!("websocket_triggers:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; let trigger = sqlx::query_as::<_, WebsocketTrigger>( r#"SELECT * FROM websocket_trigger @@ -208,6 +210,8 @@ async fn create_websocket_trigger( Path(w_id): Path, Json(ct): Json, ) -> error::Result<(StatusCode, String)> { + check_scopes(&authed, || format!("websocket_triggers:write:{}", ct.path))?; + if *CLOUD_HOSTED { return Err(error::Error::BadRequest( "WebSocket triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(), @@ -300,6 +304,7 @@ async fn update_websocket_trigger( Json(ct): Json, ) -> error::Result { let path = path.to_path(); + check_scopes(&authed, || format!("websocket_triggers:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; let filters = ct.filters.into_iter().map(SqlxJson).collect_vec(); @@ -371,8 +376,9 @@ pub async fn set_enabled( Path((w_id, path)): Path<(String, StripPath)>, Json(payload): Json, ) -> error::Result { - let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); + check_scopes(&authed, || format!("websocket_triggers:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; // important to set server_id, last_server_ping and error to NULL to stop current websocket listener let one_o = sqlx::query_scalar!( @@ -424,6 +430,8 @@ async fn delete_websocket_trigger( Path((w_id, path)): Path<(String, StripPath)>, ) -> error::Result { let path = path.to_path(); + check_scopes(&authed, || format!("websocket_triggers:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; sqlx::query!( "DELETE FROM websocket_trigger WHERE workspace_id = $1 AND path = $2", diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 980ec87d64..67c4466b17 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -102,3 +102,6 @@ tonic = { workspace = true, optional = true } [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = { optional = true, workspace = true } + +[target.'cfg(windows)'.dependencies] +sysinfo.workspace = true diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 6b720b368d..20c711c127 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -1,8 +1,7 @@ use serde::{Deserialize, Serialize}; use sqlx::PgExecutor; -use windmill_parser::asset_parser::ParseAssetsResult; -use crate::{error, scripts::ScriptLang}; +use crate::error; #[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] #[sqlx(type_name = "ASSET_KIND", rename_all = "lowercase")] @@ -10,7 +9,8 @@ use crate::{error, scripts::ScriptLang}; pub enum AssetKind { S3Object, Resource, - Variable, + // Avoid unnexpected crashes when deserializing old assets + Variable, // Deprecated } #[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] @@ -41,85 +41,28 @@ pub struct AssetUsage { pub access_type: AssetUsageAccessType, } -#[derive(Serialize, Deserialize, Debug, Clone, Hash)] -pub struct AssetWithAccessType { +#[derive(Serialize, Deserialize, Debug, Clone, Hash, sqlx::Type)] +pub struct AssetWithAltAccessType { pub path: String, pub kind: AssetKind, - pub access_type: AssetUsageAccessType, -} - -pub fn parse_assets( - input: &str, - lang: ScriptLang, -) -> anyhow::Result>>> { - let r = match lang { - ScriptLang::Python3 => windmill_parser_py::parse_assets(input), - ScriptLang::DuckDb => windmill_parser_sql::parse_assets(input).map(|a| { - a.iter() - .map(|a| ParseAssetsResult { - path: a.path.to_string(), - access_type: a.access_type, - kind: a.kind, - }) - .collect() - }), - ScriptLang::Deno | ScriptLang::Bun | ScriptLang::Nativets => { - windmill_parser_ts::parse_assets(input) - } - _ => return Ok(None), - }; - return r.map(Some); -} - -impl From for AssetKind { - fn from(kind: windmill_parser::asset_parser::AssetKind) -> Self { - match kind { - windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object, - windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, - windmill_parser::asset_parser::AssetKind::Variable => AssetKind::Variable, - } - } -} -impl From for AssetUsageAccessType { - fn from(access_type: windmill_parser::asset_parser::AssetUsageAccessType) -> Self { - match access_type { - windmill_parser::asset_parser::AssetUsageAccessType::R => AssetUsageAccessType::R, - windmill_parser::asset_parser::AssetUsageAccessType::W => AssetUsageAccessType::W, - windmill_parser::asset_parser::AssetUsageAccessType::RW => AssetUsageAccessType::RW, - } - } + pub access_type: Option, + pub alt_access_type: Option, } pub async fn insert_asset_usage<'e>( executor: impl PgExecutor<'e>, workspace_id: &str, - parsed_asset: &ParseAssetsResult, - fallback_access_types: Option<&[AssetWithAccessType]>, + asset: &AssetWithAltAccessType, usage_path: &str, usage_kind: AssetUsageKind, ) -> error::Result<()> { - let kind: AssetKind = parsed_asset.kind.into(); - let asset_alternative_access_type = || { - fallback_access_types - .as_ref() - .and_then(|v| { - v.iter() - .find(|a| a.kind == kind && a.path == parsed_asset.path) - }) - .map(|a| a.access_type) - }; - let access_type: Option = parsed_asset - .access_type - .map(Into::into) - .or_else(asset_alternative_access_type); - sqlx::query!( r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING"#, workspace_id, - parsed_asset.path, - kind as AssetKind, - access_type as Option, + asset.path, + asset.kind as AssetKind, + (asset.access_type.or(asset.alt_access_type)) as Option, usage_path, usage_kind as AssetUsageKind ) diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index b6d8f422e2..4a4cb200a8 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -18,7 +18,7 @@ use sqlx::types::Json; use sqlx::types::JsonRawValue; use crate::{ - assets::AssetWithAccessType, + assets::AssetWithAltAccessType, cache, error::Error, more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, @@ -506,7 +506,7 @@ pub enum FlowModuleValue { #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, #[serde(skip_serializing_if = "Option::is_none")] - asset_fallback_access_types: Option>, + assets: Option>, }, Identity, // Internal only, never exposed to the frontend. @@ -526,6 +526,8 @@ pub enum FlowModuleValue { concurrency_time_window_s: Option, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, + #[serde(skip_serializing_if = "Option::is_none")] + assets: Option>, }, } @@ -560,7 +562,7 @@ struct UntaggedFlowModuleValue { id: Option, default_node: Option, modules_node: Option, - asset_fallback_access_types: Option>, + assets: Option>, } impl<'de> Deserialize<'de> for FlowModuleValue { @@ -635,7 +637,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue { concurrent_limit: untagged.concurrent_limit, concurrency_time_window_s: untagged.concurrency_time_window_s, is_trigger: untagged.is_trigger, - asset_fallback_access_types: untagged.asset_fallback_access_types, + assets: untagged.assets, }), "flowscript" => Ok(FlowModuleValue::FlowScript { input_transforms: untagged.input_transforms.unwrap_or_default(), @@ -650,6 +652,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue { concurrent_limit: untagged.concurrent_limit, concurrency_time_window_s: untagged.concurrency_time_window_s, is_trigger: untagged.is_trigger, + assets: untagged.assets, }), "identity" => Ok(FlowModuleValue::Identity), other => Err(serde::de::Error::unknown_variant( @@ -785,6 +788,7 @@ pub async fn resolve_module( concurrent_limit, concurrency_time_window_s, is_trigger, + assets, } = std::mem::replace(&mut val, Identity) else { unreachable!() @@ -808,7 +812,7 @@ pub async fn resolve_module( concurrent_limit, concurrency_time_window_s, is_trigger, - asset_fallback_access_types: None, + assets, }; } ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => { diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index b619891037..c639d33b0b 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -339,7 +339,7 @@ pub async fn connect( worker_mode: bool, ) -> Result, error::Error> { use std::time::Duration; - + use sqlx::Executor; sqlx::postgres::PgPoolOptions::new() .min_connections((max_connections / 5).clamp(3, max_connections)) .max_connections(max_connections) @@ -347,13 +347,30 @@ pub async fn connect( .after_connect(move |conn, _| { if worker_mode { Box::pin(async move { - sqlx::query("SET enable_seqscan = OFF;") - .execute(conn) - .await?; + if let Err(e) = conn.execute(r#" + SET enable_seqscan = OFF; + SET statement_timeout = '5min'; + SET idle_in_transaction_session_timeout = '10min'; + SET tcp_keepalives_idle = 300; + SET tcp_keepalives_interval = 60; + SET tcp_keepalives_count = 10;"#) + .await { + tracing::error!("Error setting postgres settings: {}", e); + } Ok(()) }) } else { - Box::pin(async move { Ok(()) }) + Box::pin(async move { + if let Err(e) = conn.execute(r#" + SET statement_timeout = '5min'; + SET idle_in_transaction_session_timeout = '10min'; + SET tcp_keepalives_idle = 300; + SET tcp_keepalives_interval = 60; + SET tcp_keepalives_count = 10;"#) + .await { + tracing::error!("Error setting postgres settings: {}", e); + } + Ok(()) }) } }) .connect_with( diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 57dd7d2498..ff2f1865af 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -13,7 +13,7 @@ use std::{ }; use crate::{ - assets::AssetWithAccessType, + assets::AssetWithAltAccessType, error::{to_anyhow, Error}, utils::http_get_from_hub, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, @@ -250,6 +250,9 @@ pub struct Script { pub has_preprocessor: Option, #[serde(skip_serializing_if = "Option::is_none")] pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[sqlx(json(nullable))] + pub assets: Option>, } #[derive(Serialize, sqlx::FromRow)] @@ -349,7 +352,8 @@ pub struct NewScript { pub codebase: Option, pub has_preprocessor: Option, pub on_behalf_of_email: Option, - pub fallback_access_types: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub assets: Option>, } fn lock_deserialize<'de, D>(deserializer: D) -> Result, D::Error> diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 1f09176512..58afcc18e6 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -19,6 +19,8 @@ use std::{ str::FromStr, sync::{atomic::AtomicBool, Arc}, }; +#[cfg(windows)] +use sysinfo::System; use tokio::sync::RwLock; use uuid::Uuid; use windmill_macros::annotations; @@ -759,6 +761,7 @@ fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result< Ok(()) } +#[cfg(not(windows))] fn get_cgroupv2_path() -> Option { let cgroup_path: String = parse_file("/proc/self/cgroup")?; @@ -767,6 +770,7 @@ fn get_cgroupv2_path() -> Option { .map(|x| format!("/sys/fs/cgroup{}", x.get(1).unwrap().as_str())) } +#[cfg(not(windows))] pub fn get_vcpus() -> Option { if Path::new("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").exists() { // cgroup v1 @@ -792,6 +796,14 @@ pub fn get_vcpus() -> Option { } } +#[cfg(windows)] +pub fn get_vcpus() -> Option { + let mut sys = System::new(); + sys.refresh_cpu_all(); + (sys.cpus().len() * 100000).try_into().ok() +} + +#[cfg(not(windows))] fn get_memory_from_meminfo() -> Option { let memory_info = parse_file::("/proc/meminfo")?; if memory_info.contains("MemTotal") { @@ -808,6 +820,7 @@ fn get_memory_from_meminfo() -> Option { None } +#[cfg(not(windows))] pub fn get_memory() -> Option { let memory_limit: Option = if Path::new("/sys/fs/cgroup/memory/memory.limit_in_bytes").exists() { @@ -832,6 +845,14 @@ pub fn get_memory() -> Option { } } +#[cfg(windows)] +pub fn get_memory() -> Option { + let mut sys = System::new(); + sys.refresh_memory(); + Some(sys.total_memory() as i64) +} + +#[cfg(not(windows))] pub fn get_worker_memory_usage() -> Option { if Path::new("/sys/fs/cgroup/memory/memory.usage_in_bytes").exists() { // cgroup v1 @@ -869,6 +890,13 @@ pub fn get_worker_memory_usage() -> Option { } } +#[cfg(windows)] +pub fn get_worker_memory_usage() -> Option { + let mut sys = System::new(); + sys.refresh_memory(); + Some(sys.used_memory() as i64) +} + pub fn get_windmill_memory_usage() -> Option { #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] { diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index f6927df49f..0b8fbef7f8 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -3691,7 +3691,8 @@ pub async fn push<'c, 'd>( None, None, ), - JobPayload::RawFlowDependencies { path, flow_value } => ( + JobPayload::RawFlowDependencies { path, flow_value } => { + ( None, Some(path), None, @@ -3705,7 +3706,7 @@ pub async fn push<'c, 'd>( None, None, None, - ), + )}, JobPayload::FlowDependencies { path, dedicated_worker, version } => { // Keep inserting `value` if not all workers are updated. // Starting at `v1.440`, the value is fetched on pull from the version id. diff --git a/backend/windmill-worker/loader_builder.bun.js b/backend/windmill-worker/loader_builder.bun.js index dec0e41fed..2bf0b749f2 100644 --- a/backend/windmill-worker/loader_builder.bun.js +++ b/backend/windmill-worker/loader_builder.bun.js @@ -1,9 +1,15 @@ -const bo = await Bun.build({ - entrypoints: ["./main.ts"], - outdir: "./out", - plugins: [p], - external: ["*"], -}); +try { + await Bun.build({ + entrypoints: ["./main.ts"], + outdir: "./out", + plugins: [p], + external: ["*"], + }); +} catch (err) { + console.log(err); + console.log("Failed to build bundle"); + process.exit(1); +} const fs = require("fs/promises"); @@ -12,66 +18,58 @@ const captureVersion = import { semver } from "bun"; -if (!bo.success) { - bo.logs.forEach((l) => console.log(l)); - console.log("Failed to build bundle"); - process.exit(1); -} else { - let content = await fs.readFile("./out/main.js", { encoding: "utf8" }); - const imports = new Bun.Transpiler().scanImports( - content.replaceAll("__require", "require") - ); +let content = await fs.readFile("./out/main.js", { encoding: "utf8" }); +const imports = new Bun.Transpiler().scanImports( + content.replaceAll("__require", "require") +); - const dependencies = {}; - for (const i of imports) { - let [_, name, version] = i.path.match(captureVersion) ?? []; - if (name == undefined) { - throw Error("Unrecognized import: " + i.path); - } - if (name.startsWith("node:")) { - continue; - } - let splitted = name.split("/"); - if (splitted.length > 2) { - name = splitted.slice(0, 2).join("/"); - } - if (version == undefined) { - if (dependencies[name] == undefined) { - dependencies[name] = []; - } - } else { - if (dependencies[name] == undefined) { - dependencies[name] = [version]; - } else if (!dependencies[name].includes(version)) { - dependencies[name].push(version); - } - } +const dependencies = {}; +for (const i of imports) { + let [_, name, version] = i.path.match(captureVersion) ?? []; + if (name == undefined) { + throw Error("Unrecognized import: " + i.path); } - const resolvedDeps = {}; - for (const i in dependencies) { - const versions = dependencies[i]; - resolvedDeps[i] = - versions.length == 0 - ? "latest" - : versions.length == 1 - ? versions[0] - : reduceIntersect(versions, i); + if (name.startsWith("node:")) { + continue; } - await Bun.write( - "./package.json", - JSON.stringify({ dependencies: resolvedDeps }, null, 2) - ); - - function reduceIntersect(versions, name) { - console.log( - `multiple versions detected for ${name}: ${versions.join(", ")}` - ); - const regex = /^(?:\^|~|<|<=|>=|>)+/g; - - const r = versions - .map((x) => [x, x.replace(regex, "")]) - .sort((a, b) => semver.order(a[1], b[1]))[0][0]; - console.log(`resolved to ${r} for ${name}`); - return r; + let splitted = name.split("/"); + if (splitted.length > 2) { + name = splitted.slice(0, 2).join("/"); + } + if (version == undefined) { + if (dependencies[name] == undefined) { + dependencies[name] = []; + } + } else { + if (dependencies[name] == undefined) { + dependencies[name] = [version]; + } else if (!dependencies[name].includes(version)) { + dependencies[name].push(version); + } } } +const resolvedDeps = {}; +for (const i in dependencies) { + const versions = dependencies[i]; + resolvedDeps[i] = + versions.length == 0 + ? "latest" + : versions.length == 1 + ? versions[0] + : reduceIntersect(versions, i); +} +await Bun.write( + "./package.json", + JSON.stringify({ dependencies: resolvedDeps }, null, 2) +); + +function reduceIntersect(versions, name) { + console.log(`multiple versions detected for ${name}: ${versions.join(", ")}`); + const regex = /^(?:\^|~|<|<=|>=|>)+/g; + + const r = versions + .map((x) => [x, x.replace(regex, "")]) + .sort((a, b) => semver.order(a[1], b[1]))[0][0]; + console.log(`resolved to ${r} for ${name}`); + return r; +} diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index f724f564df..a508f3084c 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -418,17 +418,17 @@ try {{ }} -const bo = await Bun.build({{ - entrypoints: ["{job_dir}/wrapper.mjs"], - outdir: "./", - target: "node", - plugins: [p], - external: fileNames, - minify: true, - }}); - -if (!bo.success) {{ - bo.logs.forEach((l) => console.log(l)); +try {{ + await Bun.build({{ + entrypoints: ["{job_dir}/wrapper.mjs"], + outdir: "./", + target: "node", + plugins: [p], + external: fileNames, + minify: true, + }}); +}} catch(err) {{ + console.log(err); console.log("Failed to build node bundle"); process.exit(1); }} @@ -462,21 +462,21 @@ plugin(p) r#" {} -const bo = await Bun.build({{ - entrypoints: ["{job_dir}/main.ts"], - outdir: "./", - target: "{}", - plugins: [p], - external: ["electron"], - minify: {{ - identifiers: false, - syntax: true, - whitespace: false - }}, - }}); - -if (!bo.success) {{ - bo.logs.forEach((l) => console.log(l)); +try {{ + await Bun.build({{ + entrypoints: ["{job_dir}/main.ts"], + outdir: "./", + target: "{}", + plugins: [p], + external: ["electron"], + minify: {{ + identifiers: false, + syntax: true, + whitespace: false + }}, + }}); +}} catch(err) {{ + console.log(err) console.log("Failed to build node bundle"); process.exit(1); }} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7945f1eea1..ab81a628aa 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -493,7 +493,10 @@ impl JobCompletedSender { } else { unbounded_sender } - .send_async(SendResult { result: SendResultPayload::JobCompleted(jc), time: Instant::now() }) + .send_async(SendResult { + result: SendResultPayload::JobCompleted(jc), + time: Instant::now(), + }) .await .map_err(|_e| { anyhow::anyhow!("Failed to send job completed to background processor") @@ -520,9 +523,13 @@ impl JobCompletedSender { match self { Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, .. }) => { if wait_for_capacity { - sender.send_async(SendResult { result: send_result, time: Instant::now() }).await + sender + .send_async(SendResult { result: send_result, time: Instant::now() }) + .await } else { - unbounded_sender.send_async(SendResult { result: send_result, time: Instant::now() }).await + unbounded_sender + .send_async(SendResult { result: send_result, time: Instant::now() }) + .await } } Self::Http(_) => { @@ -855,13 +862,9 @@ pub fn start_interactive_worker_shell( if now.duration_since(last).as_secs() > TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION => { - Duration::from_secs( - WORKER_SHELL_NAP_TIME_DURATION, - ) - } - _ => { - Duration::from_millis(*SLEEP_QUEUE * 10) + Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION) } + _ => Duration::from_millis(*SLEEP_QUEUE * 10), }; tokio::select! { _ = tokio::time::sleep(nap_time) => { @@ -869,7 +872,7 @@ pub fn start_interactive_worker_shell( _ = killpill_rx.recv() => { break; } - } + } } Err(err) => { @@ -1329,11 +1332,15 @@ pub async fn run_worker( let mut killed_but_draining_same_worker_jobs = false; let mut killpill_rx2 = killpill_rx.resubscribe(); - + loop { let last_processing_duration_secs = last_processing_duration.load(Ordering::SeqCst); if last_processing_duration_secs > 5 { - let sleep_duration = if last_processing_duration_secs > 10 { 10 } else { 5 }; + let sleep_duration = if last_processing_duration_secs > 10 { + 10 + } else { + 5 + }; tracing::warn!(worker = %worker_name, hostname = %hostname, "last bg processor processing duration > {sleep_duration}s: {last_processing_duration_secs}s, throttling next job pull by {sleep_duration}s"); last_processing_duration.store(0, Ordering::SeqCst); tokio::time::sleep(Duration::from_secs(sleep_duration)).await; @@ -1510,16 +1517,20 @@ pub async fn run_worker( last_suspend_first = Instant::now(); } - let job = match timeout(Duration::from_secs(10), pull( - &db, - suspend_first, - &worker_name, - None, - #[cfg(feature = "benchmark")] - &mut bench, + let job = match timeout( + Duration::from_secs(10), + pull( + &db, + suspend_first, + &worker_name, + None, + #[cfg(feature = "benchmark")] + &mut bench, + ) + .warn_after_seconds(2), ) - .warn_after_seconds(2)) - .await { + .await + { Ok(job) => job, Err(e) => { tracing::error!(worker = %worker_name, hostname = %hostname, "pull timed out after 10s, sleeping for 30s: {e:?}"); @@ -1743,7 +1754,9 @@ pub async fn run_worker( match symlink_dir(&windows_parent, &windows_target).await { Ok(_) => { - tracing::info!("Successfully created directory symlink on Windows"); + tracing::info!( + "Successfully created directory symlink on Windows" + ); } Err(e) => { tracing::warn!("Failed to create symlink_dir on Windows (likely needs admin privileges or Developer Mode): {}", e); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index ebdb155465..18720456c5 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -11,9 +11,7 @@ use serde_json::{json, Value}; use sha2::Digest; use sqlx::types::Json; use uuid::Uuid; -use windmill_common::assets::{ - clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind, -}; +use windmill_common::assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind}; use windmill_common::error::Error; use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; @@ -695,6 +693,16 @@ pub async fn handle_flow_dependency_job( }) .flatten(); + let raw_deps = job + .args + .as_ref() + .map(|x| { + x.get("raw_deps") + .map(|v| serde_json::from_str::>(v.get()).ok()) + .flatten() + }) + .flatten(); + // `JobKind::FlowDependencies` job store either: // - A saved flow version `id` in the `script_hash` column. // - Preview raw flow in the `queue` or `job` table. @@ -741,6 +749,7 @@ pub async fn handle_flow_dependency_job( &nodes_to_relock, occupancy_metrics, skip_flow_update, + raw_deps, ) .await?; if !errors.is_empty() { @@ -910,6 +919,7 @@ async fn lock_modules<'c>( locks_to_reload: &Option>, occupancy_metrics: &mut OccupancyMetrics, skip_flow_update: bool, + raw_deps: Option>, // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) ) ) -> Result<( Vec, @@ -934,7 +944,7 @@ async fn lock_modules<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, - asset_fallback_access_types, + assets, } = e.get_value()? else { match e.get_value()? { @@ -963,6 +973,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, + raw_deps.clone(), )) .await?; e.value = FlowModuleValue::ForloopFlow { @@ -998,6 +1009,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, + raw_deps.clone(), )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1025,6 +1037,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, + raw_deps.clone(), )) .await?; e.value = FlowModuleValue::WhileloopFlow { @@ -1057,6 +1070,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, + raw_deps.clone(), )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1082,6 +1096,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, + raw_deps.clone(), )) .await?; errors.extend(ninner_errors); @@ -1122,12 +1137,11 @@ async fn lock_modules<'c>( continue; }; - for asset in parse_assets(&content, language)?.iter().flatten() { + for asset in assets.iter().flatten() { insert_asset_usage( &mut *tx, &job.workspace_id, asset, - asset_fallback_access_types.as_ref().map(Vec::as_slice), job_path, AssetUsageKind::Flow, ) @@ -1157,6 +1171,14 @@ async fn lock_modules<'c>( create_dir_all(job_dir).map_err(|e| { Error::ExecutionErr(format!("Error creating job dir for flow step lock: {e}")) })?; + + // If we have local lockfiles (and they are enabled) we will replace script content with lockfile and tell hander that it is raw_deps job + let (content, raw_deps) = raw_deps + .as_ref() + .and_then(|llfs| llfs.get(language.as_str())) + .map(|lock| (lock.to_owned(), true)) + .unwrap_or((content, false)); + let new_lock = capture_dependency_job( &job.id, &language, @@ -1174,7 +1196,7 @@ async fn lock_modules<'c>( "{}/flow", &path.clone().unwrap_or_else(|| job_path.to_string()) ), - false, + raw_deps, None, occupancy_metrics, ) @@ -1241,7 +1263,7 @@ async fn lock_modules<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, - asset_fallback_access_types, + assets, }); new_flow_modules.push(e); @@ -1393,6 +1415,7 @@ async fn reduce_flow<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, + assets, .. } = std::mem::replace(&mut val, Identity) else { @@ -1411,6 +1434,7 @@ async fn reduce_flow<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, + assets, }; } ForloopFlow { modules, modules_node, .. } diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index bf488b171f..6b4d6c50ee 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.507.1"; +export const VERSION = "v1.508.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/apps.ts b/cli/apps.ts index ca966e2112..324275d3ed 100644 --- a/cli/apps.ts +++ b/cli/apps.ts @@ -1,5 +1,6 @@ // deno-lint-ignore-file no-explicit-any -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { colors, Command, log, SEP, Table, yamlParseFile } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { ListableApp, Policy } from "./gen/types.gen.ts"; diff --git a/cli/auth.ts b/cli/auth.ts new file mode 100644 index 0000000000..fa50e1f3ec --- /dev/null +++ b/cli/auth.ts @@ -0,0 +1,61 @@ +// deno-lint-ignore-file no-explicit-any +import { colors, log, setClient } from "./deps.ts"; +import * as wmill from "./gen/services.gen.ts"; +import { GlobalUserInfo } from "./gen/types.gen.ts"; + +import { loginInteractive, tryGetLoginInfo } from "./login.ts"; +import { GlobalOptions } from "./types.ts"; +import { + addWorkspace, + removeWorkspace, + Workspace +} from "./workspace.ts"; + +/** + * Main authentication function - moved from context.ts to break circular dependencies + * This function maintains the original API signature from context.ts + */ +export async function requireLogin( + opts: GlobalOptions +): Promise { + // Import resolveWorkspace to avoid circular dependency at module level + const { resolveWorkspace } = await import("./context.ts"); + const workspace = await resolveWorkspace(opts); + + let token = await tryGetLoginInfo(opts); + + if (!token) { + token = workspace.token; + } + + setClient(token, workspace.remote.substring(0, workspace.remote.length - 1)); + + try { + return await wmill.globalWhoami(); + } catch (error) { + // Check for network errors and provide clearer messages + const errorMsg = error instanceof Error ? error.message : String(error); + if (errorMsg.includes('fetch') || errorMsg.includes('connection') || errorMsg.includes('ECONNREFUSED') || errorMsg.includes('refused')) { + throw new Error(`Network error: Could not connect to Windmill server at ${workspace.remote}`); + } + + log.info( + "! Could not reach API given existing credentials. Attempting to reauth..." + ); + const newToken = await loginInteractive(workspace.remote); + if (!newToken) { + throw new Error("Unauthorized: Could not authenticate with the provided credentials"); + } + + // Update workspace token + removeWorkspace(workspace.name, false, opts); + workspace.token = newToken; + addWorkspace(workspace, opts); + + setClient( + newToken, + workspace.remote.substring(0, workspace.remote.length - 1) + ); + return await wmill.globalWhoami(); + } +} diff --git a/cli/build.sh b/cli/build.sh index e228750b51..ef91f3a804 100755 --- a/cli/build.sh +++ b/cli/build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Note for mac OS users: you need to install gnu-sed with `brew install gnu-sed` and use `gsed` instead of `sed`. ./gen_wm_client.sh deno run -A dnt.ts diff --git a/cli/conf.ts b/cli/conf.ts index 342eb423af..6c4e37e3c8 100644 --- a/cli/conf.ts +++ b/cli/conf.ts @@ -108,31 +108,45 @@ export function getEffectiveSettings( ): SyncOptions { // Start with empty object - no defaults let effective = {} as SyncOptions; - + // Merge top-level settings from config (which contains user's chosen defaults) Object.keys(config).forEach(key => { if (key !== 'overrides' && config[key as keyof SyncOptions] !== undefined) { (effective as any)[key] = config[key as keyof SyncOptions]; } }); - + if (!config.overrides) { + if (repo) { + log.info(`No overrides found in wmill.yaml, using top-level settings (repository flag ignored)`); + } return effective; } - + // Construct override keys using the single format const workspaceKey = `${baseUrl}:${workspaceId}:*`; const repoKey = `${baseUrl}:${workspaceId}:${repo}`; - + + let appliedOverrides: string[] = []; + // Apply workspace-level overrides if (config.overrides[workspaceKey]) { Object.assign(effective, config.overrides[workspaceKey]); + appliedOverrides.push("workspace-level"); } - + // Apply repository-specific overrides (overrides workspace-level) if (config.overrides[repoKey]) { Object.assign(effective, config.overrides[repoKey]); + appliedOverrides.push("repository-specific"); + } else if (repo) { + // Repository was specified but no override found + log.info(`Repository override not found for "${repo}", using ${appliedOverrides.length > 0 ? appliedOverrides.join(" + ") : "top-level"} settings`); } - + + if (appliedOverrides.length > 0) { + log.info(`Applied ${appliedOverrides.join(" + ")} overrides${repo ? ` for repository "${repo}"` : ""}`); + } + return effective; } diff --git a/cli/context.ts b/cli/context.ts index c5fd7a4571..359854c880 100644 --- a/cli/context.ts +++ b/cli/context.ts @@ -58,12 +58,12 @@ export async function resolveWorkspace( if (opts.baseUrl) { if (opts.workspace && opts.token) { const normalizedBaseUrl = new URL(opts.baseUrl).toString(); // add trailing slash if not present - + // Try to find existing workspace profile by name, then by workspaceId + remote if (opts.workspace) { // Try by workspace name first let existingWorkspace = await getWorkspaceByName(opts.workspace, opts.configDir); - + // If not found by name, try to find by workspaceId + remote match if (!existingWorkspace) { const { allWorkspaces } = await import("./workspace.ts"); @@ -71,13 +71,13 @@ export async function resolveWorkspace( const matchingWorkspaces = workspaces.filter( w => w.workspaceId === opts.workspace && w.remote === normalizedBaseUrl ); - + // Due to uniqueness constraint, there can only be 0 or 1 match if (matchingWorkspaces.length === 1) { existingWorkspace = matchingWorkspaces[0]; } } - + if (existingWorkspace) { // Validate that the base URL matches the profile's remote if (existingWorkspace.remote !== normalizedBaseUrl) { @@ -95,7 +95,7 @@ export async function resolveWorkspace( }; } } - + // No existing profile found, create temporary workspace return { remote: normalizedBaseUrl, @@ -121,45 +121,6 @@ export async function resolveWorkspace( } } -export async function requireLogin( - opts: GlobalOptions -): Promise { - const workspace = await resolveWorkspace(opts); - let token = await tryGetLoginInfo(opts); - - if (!token) { - token = workspace.token; - } - - setClient(token, workspace.remote.substring(0, workspace.remote.length - 1)); - - try { - return await wmill.globalWhoami(); - } catch (error) { - // Check for network errors and provide clearer messages - const errorMsg = error instanceof Error ? error.message : String(error); - if (errorMsg.includes('fetch') || errorMsg.includes('connection') || errorMsg.includes('ECONNREFUSED') || errorMsg.includes('refused')) { - throw new Error(`Network error: Could not connect to Windmill server at ${workspace.remote}`); - } - - log.info( - "! Could not reach API given existing credentials. Attempting to reauth..." - ); - const newToken = await loginInteractive(workspace.remote); - if (!newToken) { - throw new Error("Unauthorized: Could not authenticate with the provided credentials"); - } - removeWorkspace(workspace.name, false, opts); - workspace.token = newToken; - addWorkspace(workspace, opts); - - setClient( - newToken, - workspace.remote.substring(0, workspace.remote.length - 1) - ); - return await wmill.globalWhoami(); - } -} export async function fetchVersion(baseUrl: string): Promise { const requestHeaders = new Headers(); @@ -176,13 +137,13 @@ export async function fetchVersion(baseUrl: string): Promise { new URL(new URL(baseUrl).origin + "/api/version"), { headers: requestHeaders, method: "GET" } ); - + if (!response.ok) { // Consume response body even on error to avoid resource leak await response.text(); throw new Error(`Failed to fetch version: ${response.status} ${response.statusText}`); } - + return await response.text(); } export async function tryResolveVersion( diff --git a/cli/dev.ts b/cli/dev.ts index 7cc92c45ff..658a1a057a 100644 --- a/cli/dev.ts +++ b/cli/dev.ts @@ -12,7 +12,8 @@ import { } from "./deps.ts"; import { getTypeStrFromPath, GlobalOptions } from "./types.ts"; import { ignoreF } from "./sync.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace } from "./context.ts"; import { SyncOptions, mergeConfigWithConfigFile, diff --git a/cli/flow.ts b/cli/flow.ts index 798d294db7..bf3acd67ad 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -4,10 +4,11 @@ import { Confirm, SEP, log, yamlStringify } from "./deps.ts"; import { colors, Command, Table, yamlParseFile } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { resolve, track_job } from "./script.ts"; import { defaultFlowDefinition } from "./bootstrap/flow_bootstrap.ts"; -import { generateFlowLockInternal } from "./metadata.ts"; +import { blueColor, generateFlowLockInternal } from "./metadata.ts"; import { SyncOptions, mergeConfigWithConfigFile } from "./conf.ts"; import { FSFSElement, elementsToMap, ignoreF } from "./sync.ts"; import { readInlinePathSync } from "./utils.ts"; @@ -238,15 +239,18 @@ async function run( async function generateLocks( opts: GlobalOptions & { yes?: boolean; + useRawRequirements?: boolean; } & SyncOptions, folder: string | undefined ) { + const useRawReqs = opts.useRawRequirements || Deno.env.get("USE_RAW_REQUIREMENTS") === "true"; + const workspace = await resolveWorkspace(opts); await requireLogin(opts); opts = await mergeConfigWithConfigFile(opts); if (folder) { // read script metadata file - await generateFlowLockInternal(folder, false, workspace); + await generateFlowLockInternal(folder, false, workspace, opts, undefined, undefined, useRawReqs); } else { const ignore = await ignoreF(opts); const elems = Object.keys( @@ -267,7 +271,7 @@ async function generateLocks( let hasAny = false; for (const folder of elems) { - const candidate = await generateFlowLockInternal(folder, true, workspace); + const candidate = await generateFlowLockInternal(folder, true, workspace, opts, undefined, undefined, useRawReqs); if (candidate) { hasAny = true; log.info(colors.green(`+ ${candidate}`)); @@ -289,7 +293,7 @@ async function generateLocks( return; } for (const folder of elems) { - await generateFlowLockInternal(folder, false, workspace); + await generateFlowLockInternal(folder, false, workspace, opts,undefined, undefined, useRawReqs); } } } @@ -348,6 +352,10 @@ const command = new Command() ) .arguments("[flow:file]") .option("--yes", "Skip confirmation prompt") + .option( + "-r --use-raw-requirements", + "Use raw requirements (requirements.txt, go.mod, package.json, etc) instead of generating them on the server (can also be set with USE_RAW_REQUIREMENTS=true environment variable)" + ) .option( "-i --includes ", "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)" diff --git a/cli/folder.ts b/cli/folder.ts index 5cf2812670..e60dc08903 100644 --- a/cli/folder.ts +++ b/cli/folder.ts @@ -2,7 +2,8 @@ import { colors, Command, log, SEP, Table } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts"; import { Folder } from "./gen/types.gen.ts"; diff --git a/cli/gen_wm_client.sh b/cli/gen_wm_client.sh index 21527acee9..1b36f489d9 100755 --- a/cli/gen_wm_client.sh +++ b/cli/gen_wm_client.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eou pipefail script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/cli/gitsync-settings.ts b/cli/gitsync-settings.ts index 3daeaa0415..27de3b7056 100644 --- a/cli/gitsync-settings.ts +++ b/cli/gitsync-settings.ts @@ -1,6 +1,7 @@ -import { colors, Command, log, yamlStringify } from "./deps.ts"; +import { colors, Command, Confirm, log, yamlStringify } from "./deps.ts"; import { GlobalOptions } from "./types.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace } from "./context.ts"; import * as wmill from "./gen/services.gen.ts"; import { DEFAULT_SYNC_OPTIONS, @@ -163,6 +164,147 @@ function includeTypeToSyncOptions( }; } +// Shared migration function for legacy repositories +async function handleLegacyRepositoryMigration( + selectedRepo: any, + gitSyncSettings: any, + workspace: any, + opts: { yes?: boolean }, + operationName: string = "operation" +): Promise { + if (selectedRepo.settings) { + return selectedRepo; // Already migrated + } + + // This repository is in legacy format - handle migration + if (!gitSyncSettings.include_path || !gitSyncSettings.include_type) { + throw new Error( + `Repository "${selectedRepo.git_repo_resource_path}" has legacy format but workspace-level include_path or include_type is missing. This indicates corrupted git-sync settings.` + ); + } + + const workspaceIncludePath = gitSyncSettings.include_path; + const workspaceIncludeType = gitSyncSettings.include_type; + + if (Deno.stdout.isTerminal() && !opts.yes) { + // Interactive mode - show migration prompt + console.log(colors.yellow('\n⚠️ Legacy git-sync settings detected!')); + console.log(`\nRepository "${selectedRepo.git_repo_resource_path}" has legacy settings format.`); + console.log('The new format allows per-repository filter configuration.'); + if (operationName === "push") { + console.log('This repository must be migrated before pushing settings.\n'); + } else { + console.log('\n'); + } + + console.log(colors.bold('Current workspace-level settings:')); + console.log(` Include paths: ${workspaceIncludePath.join(', ')}`); + console.log(` Include types: ${workspaceIncludeType.join(', ')}\n`); + + // Show what the migration will do + let finalIncludeType = [...workspaceIncludeType]; + if (selectedRepo.exclude_types_override && selectedRepo.exclude_types_override.length > 0) { + const originalCount = finalIncludeType.length; + finalIncludeType = finalIncludeType.filter( + type => !selectedRepo.exclude_types_override.includes(type) + ); + const excludedCount = originalCount - finalIncludeType.length; + console.log(colors.yellow(`Repository excludes ${excludedCount} types: ${selectedRepo.exclude_types_override.join(', ')}`)); + } + + console.log(colors.bold('\nAfter migration, repository will have:')); + console.log(` Include paths: ${workspaceIncludePath.join(', ')}`); + console.log(` Include types: ${finalIncludeType.join(', ')}\n`); + + const confirm = await Confirm.prompt({ + message: operationName === "push" + ? 'Do you want to migrate this repository before pushing?' + : 'Do you want to migrate this repository?', + default: true + }); + + if (!confirm) { + const message = operationName === "push" + ? '\n⚠️ Migration skipped. Cannot push to legacy repository.' + : '\n⚠️ Migration skipped. You can migrate later via the UI.'; + console.log(colors.yellow(message)); + if (operationName === "push") { + return null; // Signal to exit push operation + } + throw new Error('Migration cancelled by user'); + } + + // Perform the migration + let migratedIncludeType = [...workspaceIncludeType]; + if (selectedRepo.exclude_types_override && selectedRepo.exclude_types_override.length > 0) { + migratedIncludeType = migratedIncludeType.filter( + type => !selectedRepo.exclude_types_override.includes(type) + ); + } + + const migratedRepo = { + ...selectedRepo, + settings: { + include_path: [...workspaceIncludePath], + include_type: migratedIncludeType, + exclude_path: [], + extra_include_path: [] + } + }; + + // Remove the old field + delete migratedRepo.exclude_types_override; + + // Update the backend with migrated repository + const updatedRepositories = gitSyncSettings.repositories.map((repo: any) => { + if (repo.git_repo_resource_path === selectedRepo.git_repo_resource_path) { + return migratedRepo; + } + return repo; + }); + + await wmill.editWorkspaceGitSyncConfig({ + workspace: workspace.workspaceId, + requestBody: { + git_sync_settings: { + repositories: updatedRepositories, + // Keep workspace-level settings if other repos are still legacy + ...(gitSyncSettings.repositories.some((r: any) => r.git_repo_resource_path !== selectedRepo.git_repo_resource_path && !r.settings) && { + include_path: workspaceIncludePath, + include_type: workspaceIncludeType + }) + } + } + }); + + console.log(colors.green('\n✓ Repository migration completed successfully!')); + if (operationName === "push") { + console.log('Now proceeding with push operation...\n'); + } + return migratedRepo; + + } else { + // Non-interactive mode - show error + console.error(colors.red('\n❌ Legacy git-sync settings detected!')); + console.error(`\nRepository "${selectedRepo.git_repo_resource_path}" has legacy settings format.`); + if (operationName === "push") { + console.error('This repository must be migrated before pushing settings.'); + } + console.error('Please choose one of the following options:\n'); + console.error('1. Go to the Windmill UI > Workspace Settings > Git Sync'); + console.error(' Review and save this repository to migrate to the new format.\n'); + console.error('2. Run this command in interactive mode (with TTY) to migrate.'); + console.error(` Example: wmill gitsync-settings ${operationName}\n`); + if (operationName === "push") { + console.error('3. Pull settings first to migrate: wmill gitsync-settings pull\n'); + } else { + console.error('3. Push local settings to override backend settings:'); + console.error(' wmill gitsync-settings push\n'); + } + Deno.exit(1); + } +} + // Convert SyncOptions boolean flags to backend include_type array function syncOptionsToIncludeType(opts: SyncOptions): string[] { const includeTypes: string[] = []; @@ -278,6 +420,31 @@ function displayChanges( } } +async function selectAndLogRepository( + repositories: GitSyncRepository[], + repository?: string, +): Promise { + let selectedRepo: GitSyncRepository; + + if (repository) { + const found = repositories.find( + (r: GitSyncRepository) => + r.git_repo_resource_path === repository || + r.git_repo_resource_path === `$res:${repository}`, + ); + if (!found) { + throw new Error(`Repository ${repository} not found`); + } + selectedRepo = found; + const repoPath = selectedRepo.git_repo_resource_path.replace(/^\$res:/, ""); + log.info(colors.cyan(`Using repository: ${colors.bold(repoPath)}`)); + } else { + selectedRepo = await selectRepository(repositories); + } + + return selectedRepo; +} + async function pullGitSyncSettings( opts: GlobalOptions & { repository?: string; @@ -288,6 +455,7 @@ async function pullGitSyncSettings( replace?: boolean; override?: boolean; withBackendSettings?: string; + yes?: boolean; }, ) { const workspace = await resolveWorkspace(opts); @@ -400,22 +568,19 @@ async function pullGitSyncSettings( } // Find the repository to work with - let selectedRepo: GitSyncRepository; - if (opts.repository) { - const found = settings.git_sync.repositories.find( - (r: GitSyncRepository) => - r.git_repo_resource_path === opts.repository || - r.git_repo_resource_path === `$res:${opts.repository}`, - ); - if (!found) { - throw new Error(`Repository ${opts.repository} not found`); - } - selectedRepo = found; - } else { - selectedRepo = await selectRepository( - settings.git_sync.repositories, - ); - } + let selectedRepo = await selectAndLogRepository( + settings.git_sync.repositories, + opts.repository, + ); + + // Check if the selected repository needs migration and handle it + selectedRepo = await handleLegacyRepositoryMigration( + selectedRepo, + settings.git_sync, + workspace, + opts, + "pull" + ); // Convert backend settings to SyncOptions format const backendSyncOptions: SyncOptions = { @@ -533,7 +698,7 @@ async function pullGitSyncSettings( ); } else { if (hasChanges) { - log.info("Changes that would be made:"); + log.info("Changes that would be applied locally:"); const changes = generateChanges(normalizedCurrent, normalizedBackend); if (Object.keys(changes).length === 0) { @@ -572,7 +737,14 @@ async function pullGitSyncSettings( ); const hasConflict = !deepEqual(gitSyncBackend, gitSyncCurrent); - if (hasConflict && Deno.stdin.isTerminal()) { + if (hasConflict && !opts.yes && Deno.stdin.isTerminal()) { + // Show the diff first + log.info("Changes that would be applied locally:"); + const changes = generateChanges(currentSettings, backendSyncOptions); + if (Object.keys(changes).length > 0) { + displayChanges(changes); + } + // Interactive mode - ask user const { Select } = await import("./deps.ts"); const choice = await Select.prompt({ @@ -606,6 +778,16 @@ async function pullGitSyncSettings( repoPath, ); } + } else if (hasConflict && opts.yes) { + // --yes flag: default to override behavior for conflicts + writeMode = "override"; + const repoPath = normalizeRepoPath(selectedRepo.git_repo_resource_path); + overrideKey = constructOverrideKey( + workspace.remote, + workspace.workspaceId, + repoPath, + ); + log.info(colors.yellow("Settings conflict detected. Using --override behavior (default for --yes).")); } else if (hasConflict) { // Non-interactive mode with conflicts - show message and exit if (opts.jsonOutput) { @@ -663,11 +845,19 @@ async function pullGitSyncSettings( // Apply the settings based on write mode let updatedConfig: SyncOptions; + // Log which settings mode is being used + const repoPath = selectedRepo.git_repo_resource_path.replace(/^\$res:/, ""); + if (writeMode === "override") { + log.info(`Applied repository-specific overrides for repository "${repoPath}"`); + } else { + log.info(`Applied settings for repository "${repoPath}"`); + } + if (writeMode === "replace") { // Preserve existing local config and update only git-sync fields updatedConfig = { ...localConfig }; - // Remove overrides since we're in replace mode - delete updatedConfig.overrides; + // Clear overrides since we're in replace mode, but keep empty object for consistency + updatedConfig.overrides = {}; // Update with backend git-sync settings Object.assign(updatedConfig, backendSyncOptions); } else if (writeMode === "override" && overrideKey) { @@ -760,6 +950,7 @@ async function pushGitSyncSettings( diff?: boolean; jsonOutput?: boolean; withBackendSettings?: string; + yes?: boolean; }, ) { const workspace = await resolveWorkspace(opts); @@ -887,21 +1078,23 @@ async function pushGitSyncSettings( } // Find the repository to work with - let selectedRepo: GitSyncRepository; - if (opts.repository) { - const found = settings.git_sync.repositories.find( - (r: GitSyncRepository) => - r.git_repo_resource_path === opts.repository || - r.git_repo_resource_path === `$res:${opts.repository}`, - ); - if (!found) { - throw new Error(`Repository ${opts.repository} not found`); - } - selectedRepo = found; - } else { - selectedRepo = await selectRepository( - settings.git_sync.repositories, - ); + let selectedRepo = await selectAndLogRepository( + settings.git_sync.repositories, + opts.repository, + ); + + // Check if the selected repository needs migration and handle it + selectedRepo = await handleLegacyRepositoryMigration( + selectedRepo, + settings.git_sync, + workspace, + opts, + "push" + ); + + // If migration was cancelled, exit + if (selectedRepo === null) { + return; } // Get effective settings for this workspace/repo @@ -921,24 +1114,25 @@ async function pushGitSyncSettings( extra_include_path: effectiveSettings.extraIncludes || [], }; + // Calculate diff for all modes + const currentBackend = selectedRepo.settings; + + // Convert current backend settings to SyncOptions for user-friendly display + const currentSyncOptions: SyncOptions = { + includes: currentBackend.include_path || [], + excludes: currentBackend.exclude_path || [], + extraIncludes: currentBackend.extra_include_path || [], + ...includeTypeToSyncOptions(currentBackend.include_type || []), + }; + + const normalizedCurrent = normalizeSyncOptions(currentSyncOptions); + const normalizedEffective = normalizeSyncOptions(effectiveSettings); + const gitSyncCurrent = extractGitSyncFields(normalizedCurrent); + const gitSyncEffective = extractGitSyncFields(normalizedEffective); + const hasChanges = !deepEqual(gitSyncEffective, gitSyncCurrent); + if (opts.diff) { - // Show what would be pushed - const currentBackend = selectedRepo.settings; - - // Convert current backend settings to SyncOptions for user-friendly display - const currentSyncOptions: SyncOptions = { - includes: currentBackend.include_path || [], - excludes: currentBackend.exclude_path || [], - extraIncludes: currentBackend.extra_include_path || [], - ...includeTypeToSyncOptions(currentBackend.include_type || []), - }; - - const normalizedCurrent = normalizeSyncOptions(currentSyncOptions); - const normalizedEffective = normalizeSyncOptions(effectiveSettings); - const gitSyncCurrent = extractGitSyncFields(normalizedCurrent); - const gitSyncEffective = extractGitSyncFields(normalizedEffective); - const hasChanges = !deepEqual(gitSyncEffective, gitSyncCurrent); - + // --diff flag: show differences and exit if (opts.jsonOutput) { // Generate structured diff using the same normalized objects const structuredDiff = hasChanges @@ -957,7 +1151,7 @@ async function pushGitSyncSettings( ); } else { if (hasChanges) { - log.info("Changes that would be pushed:"); + log.info("Changes that would be pushed to Windmill:"); const changes = generateChanges( normalizedCurrent, normalizedEffective, @@ -975,6 +1169,39 @@ async function pushGitSyncSettings( return; } + // Default behavior: show changes and ask for confirmation (unless --yes is passed) + if (hasChanges) { + if (!opts.jsonOutput) { + const changes = generateChanges( + normalizedCurrent, + normalizedEffective, + ); + + if (Object.keys(changes).length === 0) { + log.info(colors.green("No changes to push")); + return; + } else { + log.info("Changes that would be pushed to Windmill:"); + displayChanges(changes); + } + } + + // Ask for confirmation unless --yes is passed or not in TTY + if (!opts.yes && Deno.stdin.isTerminal()) { + const confirmed = await Confirm.prompt({ + message: `Do you want to apply these changes to the remote?`, + default: true, + }); + + if (!confirmed) { + return; + } + } + } else { + log.info(colors.green("No changes to push")); + return; + } + if (opts.withBackendSettings) { // Skip backend update when using simulated settings if (opts.jsonOutput) { @@ -1103,6 +1330,7 @@ const command = new Command() "--with-backend-settings ", "Use provided JSON settings instead of querying backend (for testing)", ) + .option("--yes", "Skip interactive prompts and use default behavior") .action(pullGitSyncSettings as any) .command("push") .description( @@ -1118,6 +1346,7 @@ const command = new Command() "--with-backend-settings ", "Use provided JSON settings instead of querying backend (for testing)", ) + .option("--yes", "Skip interactive prompts and use default behavior") .action(pushGitSyncSettings as any); export { pullGitSyncSettings, pushGitSyncSettings }; diff --git a/cli/hub.ts b/cli/hub.ts index 6943369ec0..2f0eb0c567 100644 --- a/cli/hub.ts +++ b/cli/hub.ts @@ -2,7 +2,8 @@ import { Command, log } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace } from "./context.ts"; import { pushResourceType } from "./resource-type.ts"; import { GlobalOptions } from "./types.ts"; import { deepEqual } from "./utils.ts"; diff --git a/cli/main.ts b/cli/main.ts index aae8ea4653..0229c65b83 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -65,7 +65,7 @@ export { // } // }); -export const VERSION = "1.507.1"; +export const VERSION = "1.508.0"; const command = new Command() .name("wmill") @@ -143,6 +143,7 @@ const command = new Command() skipFolders: DEFAULT_SYNC_OPTIONS.skipFolders, includeSchedules: DEFAULT_SYNC_OPTIONS.includeSchedules, includeTriggers: DEFAULT_SYNC_OPTIONS.includeTriggers, + overrides: {}, }; await Deno.writeTextFile( @@ -159,9 +160,8 @@ const command = new Command() // Check for backend git-sync settings unless --use-default is specified if (!opts.useDefault) { try { - const { requireLogin, resolveWorkspace } = await import( - "./context.ts" - ); + const { requireLogin } = await import("./auth.ts"); + const { resolveWorkspace } = await import("./context.ts"); // Check if user has workspace configured const { getActiveWorkspace } = await import( diff --git a/cli/metadata.ts b/cli/metadata.ts index d2d37aa22d..dcf3cdef3b 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -14,9 +14,9 @@ import { } from "./bootstrap/script_bootstrap.ts"; import { Workspace } from "./workspace.ts"; import { SchemaProperty } from "./bootstrap/common.ts"; -import { ScriptLanguage } from "./script_common.ts"; +import { languagesWithRawReqsSupport, LanguageWithRawReqsSupport, ScriptLanguage } from "./script_common.ts"; import { inferContentTypeFromFilePath } from "./script_common.ts"; -import { GlobalDeps, exts } from "./script.ts"; +import { GlobalDeps, exts, findGlobalDeps } from "./script.ts"; import { FSFSElement, extractInlineScriptsForFlows, @@ -37,43 +37,16 @@ export class LockfileGenerationError extends Error { } } -export async function generateAllMetadata() {} +export async function generateAllMetadata() { } function findClosestRawReqs( - lang: "bun" | "python3" | "php" | "go" | undefined, + lang: LanguageWithRawReqsSupport | undefined, remotePath: string, - globalDeps: GlobalDeps + globalDeps: GlobalDeps, ): string | undefined { let bestCandidate: { k: string; v: string } | undefined = undefined; - if (lang == "bun") { - Object.entries(globalDeps.pkgs).forEach(([k, v]) => { - if ( - remotePath.startsWith(k) && - k.length >= (bestCandidate?.k ?? "").length - ) { - bestCandidate = { k, v }; - } - }); - } else if (lang == "python3") { - Object.entries(globalDeps.reqs).forEach(([k, v]) => { - if ( - remotePath.startsWith(k) && - k.length >= (bestCandidate?.k ?? "").length - ) { - bestCandidate = { k, v }; - } - }); - } else if (lang == "php") { - Object.entries(globalDeps.composers).forEach(([k, v]) => { - if ( - remotePath.startsWith(k) && - k.length >= (bestCandidate?.k ?? "").length - ) { - bestCandidate = { k, v }; - } - }); - } else if (lang == "go") { - Object.entries(globalDeps.goMods).forEach(([k, v]) => { + if (lang) { + Object.entries(globalDeps.get(lang) ?? {}).forEach(([k, v]) => { if ( remotePath.startsWith(k) && k.length >= (bestCandidate?.k ?? "").length @@ -87,12 +60,25 @@ function findClosestRawReqs( } const TOP_HASH = "__flow_hash"; -async function generateFlowHash(folder: string) { +async function generateFlowHash( + rawReqs: Record | undefined, + folder: string, + defaultTs: "bun" | "deno" | undefined +) { const elems = await FSFSElement(path.join(Deno.cwd(), folder), [], true); const hashes: Record = {}; for await (const f of elems.getChildren()) { if (exts.some((e) => f.path.endsWith(e))) { - hashes[f.path] = await generateHash(await f.getContentText()); + let reqs: string | undefined; + if (rawReqs) { + // Get language name from path + const lang = inferContentTypeFromFilePath(f.path, defaultTs); + // Get lock for that language + [, reqs] = Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? []; + } + + // Embed lock into hash + hashes[f.path] = await generateHash(await f.getContentText() + (reqs ?? "")); } } return { ...hashes, [TOP_HASH]: await generateHash(JSON.stringify(hashes)) }; @@ -101,9 +87,13 @@ export async function generateFlowLockInternal( folder: string, dryRun: boolean, workspace: Workspace, + opts: GlobalOptions & { + defaultTs?: "bun" | "deno"; + }, justUpdateMetadataLock?: boolean, - noStaleMessage?: boolean -): Promise { + noStaleMessage?: boolean, + useRawReqs?: boolean, +): Promise { if (folder.endsWith(SEP)) { folder = folder.substring(0, folder.length - 1); } @@ -114,7 +104,24 @@ export async function generateFlowLockInternal( log.info(`Generating lock for flow ${folder} at ${remote_path}`); } - let hashes = await generateFlowHash(folder); + let rawReqs: Record | undefined = undefined; + if (useRawReqs) { + // Find all dependency files in the workspace + const globalDeps = await findGlobalDeps(); + + // Find closest dependency files for this flow + rawReqs = {}; + + // TODO: PERF: Only include raw reqs for the languages that are in the flow + languagesWithRawReqsSupport.map((lang) => { + const dep = findClosestRawReqs(lang, folder, globalDeps); + if (dep) { + // @ts-ignore + rawReqs[lang.language] = dep; + } + }); + } + let hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs); const conf = await readLockfile(); if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) { @@ -128,8 +135,17 @@ export async function generateFlowLockInternal( return remote_path; } + if (useRawReqs) { + log.warn("If using local lockfiles, following redeployments from Web App will inevitably override generated lockfiles by CLI. To maintain your script's lockfiles you will need to redeploy only from CLI. (Behavior is subject to change)") + log.info( + (await blueColor())( + `Found raw requirements (${languagesWithRawReqsSupport.map((l) => l.rrFilename).join("/")}) for ${folder}, using it`, + ), + ); + } + const flowValue = (await yamlParseFile( - folder! + SEP + "flow.yaml" + folder! + SEP + "flow.yaml", )) as FlowFile; if (!justUpdateMetadataLock) { @@ -148,26 +164,40 @@ export async function generateFlowLockInternal( replaceInlineScripts( flowValue.value.modules, folder + SEP!, - changedScripts + changedScripts, ); + //removeChangedLocks - flowValue.value = await updateFlow(workspace, flowValue.value, remote_path); + flowValue.value = await updateFlow( + workspace, + flowValue.value, + remote_path, + rawReqs, + ); const inlineScripts = extractInlineScriptsForFlows( flowValue.value.modules, - newPathAssigner("bun") + newPathAssigner(opts.defaultTs ?? "bun"), ); inlineScripts .filter((s) => s.path.endsWith(".lock")) .forEach((s) => { Deno.writeTextFileSync( Deno.cwd() + SEP + folder + SEP + s.path, - s.content + s.content, ); }); + + // Overwrite `flow.yaml` with the new lockfile references + await Deno.writeTextFile( + Deno.cwd() + SEP + folder + SEP + "flow.yaml", + yamlStringify( + flowValue as Record + ) + ); } - hashes = await generateFlowHash(folder); + hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs); for (const [path, hash] of Object.entries(hashes)) { await updateMetadataGlobalLock(folder, hash, path); @@ -176,7 +206,7 @@ export async function generateFlowLockInternal( } // on windows, when using powershell, blue is not readable -async function blueColor(): Promise<(x: string) => void> { +export async function blueColor(): Promise<(x: string) => void> { const isWin = await getIsWin(); return isWin ? colors.black : colors.blue; } @@ -193,7 +223,7 @@ export async function generateScriptMetadataInternal( noStaleMessage: boolean, globalDeps: GlobalDeps, codebases: SyncCodebase[], - justUpdateMetadataLock?: boolean + justUpdateMetadataLock?: boolean, ): Promise { const remotePath = scriptPath .substring(0, scriptPath.indexOf(".")) @@ -201,23 +231,26 @@ export async function generateScriptMetadataInternal( const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs); + const rrLang = languagesWithRawReqsSupport.find((l) => language == l.language); + const rawReqs = findClosestRawReqs( - language as "bun" | "python3" | "php" | "go" | undefined, + rrLang, scriptPath, - globalDeps + globalDeps, ); - if (rawReqs) { + + if (rawReqs && rrLang) { log.info( (await blueColor())( - `Found raw requirements (package.json/requirements.txt/composer.json/go.mod) for ${scriptPath}, using it` - ) + `Found raw requirements (${rrLang.rrFilename}) for ${scriptPath}, using it`, + ), ); } const metadataWithType = await parseMetadataFile( remotePath, undefined, globalDeps, - codebases + codebases, ); // read script content @@ -229,7 +262,7 @@ export async function generateScriptMetadataInternal( if (await checkifMetadataUptodate(remotePath, hash, undefined)) { if (!noStaleMessage) { log.info( - colors.green(`Script ${remotePath} metadata is up-to-date, skipping`) + colors.green(`Script ${remotePath} metadata is up-to-date, skipping`), ); } return; @@ -251,7 +284,7 @@ export async function generateScriptMetadataInternal( scriptContent, language, metadataParsedContent, - scriptPath + scriptPath, ); } @@ -265,7 +298,7 @@ export async function generateScriptMetadataInternal( language, remotePath, metadataParsedContent, - rawReqs + rawReqs, ); } else { metadataParsedContent.lock = ""; @@ -286,7 +319,7 @@ export async function generateScriptMetadataInternal( hash = await generateScriptHash( rawReqs, scriptContent, - metadataContentUsedForHash + metadataContentUsedForHash, ); await updateMetadataGlobalLock(remotePath, hash); if (!justUpdateMetadataLock) { @@ -299,14 +332,14 @@ export async function updateScriptSchema( scriptContent: string, language: ScriptLanguage, metadataContent: Record, - path: string + path: string, ): Promise { // infer schema from script content and update it inplace const result = await inferSchema( language, scriptContent, metadataContent.schema, - path + path, ); metadataContent.schema = result.schema; if (result.has_preprocessor) { @@ -327,15 +360,12 @@ async function updateScriptLock( language: ScriptLanguage, remotePath: string, metadataContent: Record, - rawDeps: string | undefined + rawDeps: string | undefined, ): Promise { if ( !( - language == "bun" || - language == "python3" || - language == "go" || + languagesWithRawReqsSupport.some((l) => l.language == language) || language == "deno" || - language == "php" || language == "rust" || language == "ansible" ) @@ -363,7 +393,7 @@ async function updateScriptLock( raw_deps: rawDeps, entrypoint: remotePath, }), - } + }, ); let responseText = "reading response failed"; @@ -374,11 +404,11 @@ async function updateScriptLock( if (lock === undefined) { if (response?.["error"]?.["message"]) { throw new LockfileGenerationError( - `Failed to generate lockfile: ${response?.["error"]?.["message"]}` + `Failed to generate lockfile: ${response?.["error"]?.["message"]}`, ); } throw new LockfileGenerationError( - `Failed to generate lockfile: ${JSON.stringify(response, null, 2)}` + `Failed to generate lockfile: ${JSON.stringify(response, null, 2)}`, ); } const lockPath = remotePath + ".script.lock"; @@ -390,7 +420,7 @@ async function updateScriptLock( if (await Deno.stat(lockPath)) { await Deno.remove(lockPath); } - } catch {} + } catch { } metadataContent.lock = ""; } } catch (e) { @@ -398,7 +428,7 @@ async function updateScriptLock( throw e; } throw new LockfileGenerationError( - `Failed to generate lockfile:${rawResponse.statusText}, ${responseText}, ${e}` + `Failed to generate lockfile:${rawResponse.statusText}, ${responseText}, ${e}`, ); } } @@ -406,24 +436,48 @@ async function updateScriptLock( export async function updateFlow( workspace: Workspace, flow_value: FlowValue, - remotePath: string + remotePath: string, + rawDeps?: Record ): Promise { - // generate the script lock running a dependency job in Windmill and update it inplace - // TODO: update this once the client is released - const rawResponse = await fetch( - `${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`, - { - method: "POST", - headers: { - Cookie: `token=${workspace.token}`, - "Content-Type": "application/json", + let rawResponse; + + if (rawDeps != undefined) { + log.info(colors.blue("Using raw requirements for flow dependencies")); + + // generate the script lock running a dependency job in Windmill and update it inplace + rawResponse = await fetch( + `${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`, + { + method: "POST", + headers: { + Cookie: `token=${workspace.token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + flow_value, + path: remotePath, + use_local_lockfiles: true, + raw_deps: rawDeps, + }), }, - body: JSON.stringify({ - flow_value, - path: remotePath, - }), - } - ); + ); + } else { + // Standard dependency resolution on the server + rawResponse = await fetch( + `${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`, + { + method: "POST", + headers: { + Cookie: `token=${workspace.token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + flow_value, + path: remotePath, + }), + }, + ); + } let responseText = "reading response failed"; try { @@ -435,20 +489,20 @@ export async function updateFlow( const msg = (res as any)?.["error"]?.["message"]; if (msg) { throw new LockfileGenerationError( - `Failed to generate lockfile: ${msg}` + `Failed to generate lockfile: ${msg}`, ); } throw new LockfileGenerationError( - `Failed to generate lockfile: ${rawResponse.statusText}, ${responseText}` + `Failed to generate lockfile: ${rawResponse.statusText}, ${responseText}`, ); } return (res as any).updated_flow_value; } catch (e) { try { responseText = await rawResponse.text(); - } catch {} + } catch { } throw new Error( - `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}` + `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`, ); } } @@ -460,7 +514,7 @@ export async function inferSchema( language: ScriptLanguage, content: string, currentSchema: any, - path: string + path: string, ): Promise<{ schema: any; has_preprocessor: boolean | undefined; @@ -586,8 +640,8 @@ export async function inferSchema( if (inferedSchema.type == "Invalid") { log.info( colors.yellow( - `Script ${path} invalid, it cannot be parsed to infer schema.` - ) + `Script ${path} invalid, it cannot be parsed to infer schema.`, + ), ); return { schema: defaultScriptMetadata().schema, @@ -601,7 +655,7 @@ export async function inferSchema( } currentSchema.required = []; const oldProperties = JSON.parse( - JSON.stringify(currentSchema?.properties ?? {}) + JSON.stringify(currentSchema?.properties ?? {}), ); currentSchema.properties = {}; @@ -612,7 +666,7 @@ export async function inferSchema( currentSchema.properties[arg.name] = oldProperties[arg.name]; } currentSchema.properties[arg.name] = sortObject( - currentSchema.properties[arg.name] + currentSchema.properties[arg.name], ); argSigToJsonSchemaType(arg.typ, currentSchema.properties[arg.name]); @@ -639,7 +693,7 @@ function sortObject(obj: any): any { ...acc, [key]: obj[key], }), - {} + {}, ); } @@ -649,24 +703,24 @@ export function argSigToJsonSchemaType( | string | { resource: string | null } | { - list: - | (string | { object: { key: string; typ: any }[] }) - | { str: any } - | { object: { key: string; typ: any }[] } - | null; - } + list: + | (string | { object: { key: string; typ: any }[] }) + | { str: any } + | { object: { key: string; typ: any }[] } + | null; + } | { dynselect: string } | { str: string[] | null } | { object: { key: string; typ: any }[] } | { - oneof: [ - { - label: string; - properties: { key: string; typ: any }[]; - } - ]; - }, - oldS: SchemaProperty + oneof: [ + { + label: string; + properties: { key: string; typ: any }[]; + }, + ]; + }, + oldS: SchemaProperty, ): void { const newS: SchemaProperty = { type: "" }; if (t === "int") { @@ -858,7 +912,9 @@ export function replaceLock(o?: { lock?: string | string[] }) { o.lock = readInlinePathSync(lockPath); } catch (e) { log.info( - colors.yellow(`Failed to read lockfile, doing as if it was empty: ${e}`) + colors.yellow( + `Failed to read lockfile, doing as if it was empty: ${e}`, + ), ); o.lock = ""; } @@ -868,13 +924,13 @@ export async function parseMetadataFile( scriptPath: string, generateMetadataIfMissing: | (GlobalOptions & { - path: string; - workspaceRemote: Workspace; - schemaOnly?: boolean; - }) + path: string; + workspaceRemote: Workspace; + schemaOnly?: boolean; + }) | undefined, globalDeps: GlobalDeps, - codebases: SyncCodebase[] + codebases: SyncCodebase[], ): Promise<{ isJson: boolean; payload: any; path: string }> { let metadataFilePath = scriptPath + ".script.json"; try { @@ -900,14 +956,14 @@ export async function parseMetadataFile( // no metadata file at all. Create it log.info( (await blueColor())( - `Creating script metadata file for ${metadataFilePath}` - ) + `Creating script metadata file for ${metadataFilePath}`, + ), ); metadataFilePath = scriptPath + ".script.yaml"; let scriptInitialMetadata = defaultScriptMetadata(); const scriptInitialMetadataYaml = yamlStringify( scriptInitialMetadata as Record, - yamlOptions + yamlOptions, ); await Deno.writeTextFile(metadataFilePath, scriptInitialMetadataYaml, { createNew: true, @@ -916,8 +972,8 @@ export async function parseMetadataFile( if (generateMetadataIfMissing) { log.info( (await blueColor())( - `Generating lockfile and schema for ${metadataFilePath}` - ) + `Generating lockfile and schema for ${metadataFilePath}`, + ), ); try { await generateScriptMetadataInternal( @@ -928,10 +984,10 @@ export async function parseMetadataFile( false, globalDeps, codebases, - false + false, ); scriptInitialMetadata = (await yamlParseFile( - metadataFilePath + metadataFilePath, )) as ScriptMetadata; if (!generateMetadataIfMissing.schemaOnly) { replaceLock(scriptInitialMetadata); @@ -939,8 +995,8 @@ export async function parseMetadataFile( } catch (e) { log.info( colors.yellow( - `Failed to generate lockfile and schema for ${metadataFilePath}: ${e}` - ) + `Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`, + ), ); } } @@ -979,7 +1035,7 @@ export async function checkifMetadataUptodate( path: string, hash: string, conf: Lock | undefined, - subpath?: string + subpath?: string, ) { if (!conf) { conf = await readLockfile(); @@ -995,17 +1051,17 @@ export async function checkifMetadataUptodate( export async function generateScriptHash( rawReqs: string | undefined, scriptContent: string, - newMetadataContent: string + newMetadataContent: string, ) { return await generateHash( - (rawReqs ?? "") + scriptContent + newMetadataContent + (rawReqs ?? "") + scriptContent + newMetadataContent, ); } export async function updateMetadataGlobalLock( path: string, hash: string, - subpath?: string + subpath?: string, ): Promise { const conf = await readLockfile(); if (!conf?.locks) { @@ -1024,6 +1080,6 @@ export async function updateMetadataGlobalLock( } await Deno.writeTextFile( WMILL_LOCKFILE, - yamlStringify(conf as Record, yamlOptions) + yamlStringify(conf as Record, yamlOptions), ); } diff --git a/cli/resource-type.ts b/cli/resource-type.ts index 7a25c5a36e..aef94d2249 100644 --- a/cli/resource-type.ts +++ b/cli/resource-type.ts @@ -5,7 +5,8 @@ import { parseFromFile, removeType, } from "./types.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace } from "./context.ts"; import { colors, Command, log, Table } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { ResourceType } from "./gen/types.gen.ts"; diff --git a/cli/resource.ts b/cli/resource.ts index 1b65cf9420..0d620f68e3 100644 --- a/cli/resource.ts +++ b/cli/resource.ts @@ -5,7 +5,8 @@ import { parseFromFile, removeType, } from "./types.ts"; -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { colors, Command, log, SEP, Table } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { Resource } from "./gen/types.gen.ts"; diff --git a/cli/schedule.ts b/cli/schedule.ts index 77052e371d..34ec378a18 100644 --- a/cli/schedule.ts +++ b/cli/schedule.ts @@ -1,6 +1,7 @@ // deno-lint-ignore-file no-explicit-any import { colors, Command, log, SEP, Table } from "./deps.ts"; -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import * as wmill from "./gen/services.gen.ts"; import { diff --git a/cli/script.ts b/cli/script.ts index 87e4f85354..8ae5e3f6e6 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -1,6 +1,7 @@ // deno-lint-ignore-file no-explicit-any import { GlobalOptions } from "./types.ts"; -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { colors, Command, @@ -26,8 +27,10 @@ import { parseMetadataFile, } from "./metadata.ts"; import { + LanguageWithRawReqsSupport, ScriptLanguage, inferContentTypeFromFilePath, + languagesWithRawReqsSupport, } from "./script_common.ts"; import { elementsToMap, @@ -865,43 +868,35 @@ async function bootstrap( ); } -export type GlobalDeps = { - pkgs: Record; - reqs: Record; - composers: Record; - goMods: Record; -}; +export type GlobalDeps = Map>; + export async function findGlobalDeps(): Promise { - const pkgs: { [key: string]: string } = {}; - const reqs: { [key: string]: string } = {}; - const composers: { [key: string]: string } = {}; - const goMods: { [key: string]: string } = {}; + var globalDeps: GlobalDeps = new Map(); const els = await FSFSElement(Deno.cwd(), [], false); for await (const entry of readDirRecursiveWithIgnore((p, isDir) => { p = SEP + p; return ( !isDir && - !( - p.endsWith(SEP + "package.json") || - p.endsWith(SEP + "requirements.txt") || - p.endsWith(SEP + "composer.json") || - p.endsWith(SEP + "go.mod") + // Skip if the filename is not one of lockfile names + !(languagesWithRawReqsSupport.some( + lockfile => + p.endsWith(SEP + lockfile.rrFilename)) ) ); }, els)) { if (entry.isDirectory || entry.ignored) continue; const content = await entry.getContentText(); - if (entry.path.endsWith("package.json")) { - pkgs[entry.path.substring(0, entry.path.length - "package.json".length)] = content; - } else if (entry.path.endsWith("requirements.txt")) { - reqs[entry.path.substring(0, entry.path.length - "requirements.txt".length)] = content; - } else if (entry.path.endsWith("composer.json")) { - composers[entry.path.substring(0, entry.path.length - "composer.json".length)] = content; - } else if (entry.path.endsWith("go.mod")) { - goMods[entry.path.substring(0, entry.path.length - "go.mod".length)] = content; - } + + // Iterate over available languages to find which lockfile + languagesWithRawReqsSupport.map((lock) => { + if (entry.path.endsWith(lock.rrFilename)){ + const current = globalDeps.get(lock) ?? {}; + current[entry.path.substring(0, entry.path.length - lock.rrFilename.length)] = content; + globalDeps.set(lock, current); + } + }); } - return { pkgs, reqs, composers, goMods }; + return globalDeps; } async function generateMetadata( opts: GlobalOptions & { diff --git a/cli/script_common.ts b/cli/script_common.ts index 39906e7bb9..85efea9786 100644 --- a/cli/script_common.ts +++ b/cli/script_common.ts @@ -22,6 +22,23 @@ export type ScriptLanguage = | "java"; // for related places search: ADD_NEW_LANG +// To make language support raw requirements: +// 1. Add value here +// 2. Modify backend to allow raw deps +export type LanguageWithRawReqsSupport = + | { language: "bun", rrFilename /** (raw requirements filename) */: "package.json" } + // TODO: Add `requirements.in` - more intuitive and reflects better what actually happens + | { language: "python3", rrFilename: "requirements.txt" } + | { language: "php", rrFilename: "composer.json" } + | { language: "go", rrFilename: "go.mod" }; + +export const languagesWithRawReqsSupport: LanguageWithRawReqsSupport[] = [ + { language: "bun", rrFilename: "package.json" }, + { language: "python3", rrFilename: "requirements.txt" }, + { language: "php", rrFilename: "composer.json" }, + { language: "go", rrFilename: "go.mod" }, +] as const; + export function inferContentTypeFromFilePath( contentPath: string, defaultTs: "bun" | "deno" | undefined diff --git a/cli/sync.ts b/cli/sync.ts index 8ac9306a11..4f9cb8a715 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -1,4 +1,5 @@ -import { fetchVersion, requireLogin, resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { fetchVersion, resolveWorkspace } from "./context.ts"; import { colors, Command, @@ -55,43 +56,8 @@ function mergeCliWithEffectiveOptions 0) { log.info( @@ -1686,6 +1654,7 @@ export async function push(opts: GlobalOptions & SyncOptions & { repository?: st change, true, workspace, + opts, false, true ); diff --git a/cli/test/gitsync_settings_features.test.ts b/cli/test/gitsync_settings_features.test.ts index b4ad26e9b8..c5a7197bc3 100644 --- a/cli/test/gitsync_settings_features.test.ts +++ b/cli/test/gitsync_settings_features.test.ts @@ -55,7 +55,7 @@ excludes: []`); // Read updated config const updatedConfig = await Deno.readTextFile(`${tempDir}/wmill.yaml`); const backendUrl = new URL(backend.baseUrl).toString(); - + // Should create workspace wildcard override assertStringIncludes(updatedConfig, `'${backendUrl}:${backend.workspace}:*':`); assertStringIncludes(updatedConfig, "overrides:"); @@ -109,14 +109,14 @@ skipVariables: false`); // Read updated config const updatedConfig = await Deno.readTextFile(`${tempDir}/wmill.yaml`); - + // Should update top-level settings, not create overrides assertStringIncludes(updatedConfig, "includes:\n - f/special/**"); assertStringIncludes(updatedConfig, "excludes:\n - '*.test.ts'"); assertStringIncludes(updatedConfig, "extraIncludes:\n - g/**"); - - // Should NOT have overrides section - assertEquals(updatedConfig.includes("overrides:"), false, "Default mode should not create overrides"); + + // Should have empty overrides section for consistency + assertStringIncludes(updatedConfig, "overrides: {}", "Should have empty overrides section for consistency"); }); }); @@ -168,10 +168,10 @@ skipResources: false`); ], tempDir); assertEquals(result.code, 0); - + // Should show differences - assertStringIncludes(result.stdout, "Changes that would be made:"); + assertStringIncludes(result.stdout, "Changes that would be applied locally:"); // Should show the change for skipResources (ignoring ANSI color codes) assertStringIncludes(result.stdout, "skipResources:"); }); -}); \ No newline at end of file +}); diff --git a/cli/test/init_no_git_sync.test.ts b/cli/test/init_no_git_sync.test.ts index 2be95fe24e..94dad104ff 100644 --- a/cli/test/init_no_git_sync.test.ts +++ b/cli/test/init_no_git_sync.test.ts @@ -126,8 +126,8 @@ Deno.test("Init: --use-backend flag applies git-sync settings", async () => { assertStringIncludes(wmillYaml, "*.test.ts", "Should include backend's exclude_path"); assertStringIncludes(wmillYaml, "g/**", "Should include backend's extra_include_path"); - // Should NOT have overrides section since we're starting fresh - assertEquals(wmillYaml.includes("overrides:"), false, "Init should not create overrides section"); + // Should have empty overrides section for consistency + assertStringIncludes(wmillYaml, "overrides: {}"); }); }); @@ -180,6 +180,6 @@ Deno.test("Init: --use-default bypasses backend settings check", async () => { // Should NOT have backend-specific settings assertEquals(wmillYaml.includes("f/should-be-ignored/**"), false, "Should not include backend settings"); - assertEquals(wmillYaml.includes("overrides:"), false, "Should not create overrides when using defaults"); + assertStringIncludes(wmillYaml, "overrides: {}", "Should have empty overrides section for consistency"); }); }); \ No newline at end of file diff --git a/cli/test/multi_instance_workspace.test.ts b/cli/test/multi_instance_workspace.test.ts index bfcc86a3b3..97ad4f3ec4 100644 --- a/cli/test/multi_instance_workspace.test.ts +++ b/cli/test/multi_instance_workspace.test.ts @@ -24,7 +24,7 @@ Deno.test("Multi-Instance: gitsync-settings pull with new format", async () => { await withContainerizedBackend(async (backend, tempDir) => { // Set up workspace profile await setupWorkspaceProfile(backend, "multi_instance_test"); - + // Create wmill.yaml with new format overrides for different instances const backendUrl = new URL(backend.baseUrl).toString(); // Normalize URL await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun @@ -38,22 +38,22 @@ overrides: includeTriggers: true includeSchedules: true skipVariables: true - + # Different instance (won't match) "https://app.windmill.dev/:${backend.workspace}:u/test/test_repo": includeTriggers: false includeSchedules: false skipVariables: false`); - + // Pull settings - should use the matching instance override (skipVariables: true) const pullResult = await backend.runCLICommand([ 'gitsync-settings', 'pull', '--repository', 'u/test/test_repo', '--diff' ], tempDir, "multi_instance_test"); - + assertEquals(pullResult.code, 0); - assertStringIncludes(pullResult.stdout, "Changes that would be made:"); + assertStringIncludes(pullResult.stdout, "Changes that would be applied locally:"); // includeSchedules should show as a change since backend default is false assertStringIncludes(pullResult.stdout, "includeSchedules"); }); @@ -61,9 +61,9 @@ overrides: Deno.test("Multi-Instance: gitsync-settings push with overrides", async () => { await withContainerizedBackend(async (backend, tempDir) => { - // Set up workspace profile + // Set up workspace profile await setupWorkspaceProfile(backend, "push_override_test"); - + // Create wmill.yaml with specific settings that differ from backend defaults const backendUrl = new URL(backend.baseUrl).toString(); await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun @@ -78,16 +78,16 @@ overrides: "${backendUrl}:${backend.workspace}:u/test/test_repo": includeSchedules: true skipVariables: true`); - + // Push settings - should show changes because includeSchedules differs from backend const pushResult = await backend.runCLICommand([ 'gitsync-settings', 'push', '--repository', 'u/test/test_repo', '--diff' ], tempDir, "push_override_test"); - + assertEquals(pushResult.code, 0); - assertStringIncludes(pushResult.stdout, "Changes that would be pushed:"); + assertStringIncludes(pushResult.stdout, "Changes that would be pushed to Windmill:"); assertStringIncludes(pushResult.stdout, "includeSchedules"); }); }); @@ -95,7 +95,7 @@ overrides: Deno.test("Multi-Instance: sync with repository-specific overrides", async () => { await withContainerizedBackend(async (backend, tempDir) => { await setupWorkspaceProfile(backend, "my-workspace_123"); - + const backendUrl = new URL(backend.baseUrl).toString(); await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: @@ -105,22 +105,22 @@ excludes: [] overrides: "${backendUrl}:${backend.workspace}:u/test/test_repo": skipApps: true`); - + const result = await backend.runCLICommand([ 'sync', 'pull', '--repository', 'u/test/test_repo', '--dry-run', '--json-output' ], tempDir, "my-workspace_123"); - + assertEquals(result.code, 0); - + const data = parseJsonFromCLIOutput(result.stdout); - + // Test is designed to verify that the new format works correctly - + // The test app should NOT appear in changes because skipApps: true - const hasTestApp = (data.changes || []).some((change: any) => + const hasTestApp = (data.changes || []).some((change: any) => change.path?.includes('f/test_dashboard') ); assertEquals(hasTestApp, false, "Test app should be skipped due to skipApps override"); @@ -130,7 +130,7 @@ overrides: Deno.test("Multi-Instance: auto-detection of single repository override", async () => { await withContainerizedBackend(async (backend, tempDir) => { await setupWorkspaceProfile(backend, "auto_detect_test"); - + const backendUrl = new URL(backend.baseUrl).toString(); await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: @@ -142,21 +142,21 @@ overrides: "${backendUrl}:${backend.workspace}:u/test/test_repo": skipApps: true includeSchedules: true`); - + // Don't specify --repository, it should auto-detect const result = await backend.runCLICommand([ 'sync', 'pull', '--dry-run', '--json-output' ], tempDir, "auto_detect_test"); - + assertEquals(result.code, 0); assertStringIncludes(result.stdout, "Auto-selected repository: u/test/test_repo"); - + const data = parseJsonFromCLIOutput(result.stdout); - + // The test app should NOT appear because of auto-detected skipApps: true - const hasTestApp = (data.changes || []).some((change: any) => + const hasTestApp = (data.changes || []).some((change: any) => change.path?.includes('f/test_dashboard') ); assertEquals(hasTestApp, false, "Test app should be skipped due to auto-detected override"); @@ -166,7 +166,7 @@ overrides: Deno.test("Multi-Instance: workspace wildcards with new format", async () => { await withContainerizedBackend(async (backend, tempDir) => { await setupWorkspaceProfile(backend, "wildcard_test"); - + const backendUrl = new URL(backend.baseUrl).toString(); await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: @@ -178,22 +178,22 @@ overrides: "${backendUrl}:${backend.workspace}:*": skipVariables: true skipResources: true`); - + const result = await backend.runCLICommand([ 'sync', 'pull', '--repository', 'u/test/test_repo', '--dry-run', '--json-output' ], tempDir, "wildcard_test"); - + assertEquals(result.code, 0); - + const data = parseJsonFromCLIOutput(result.stdout); - + // Variables should be skipped due to wildcard override - const hasTestVariable = (data.changes || []).some((change: any) => + const hasTestVariable = (data.changes || []).some((change: any) => change.path?.includes('u/admin/test_config.variable.yaml') ); assertEquals(hasTestVariable, false, "Variables should be skipped due to wildcard override"); }); -}); \ No newline at end of file +}); diff --git a/cli/test/sync_config_resolution.test.ts b/cli/test/sync_config_resolution.test.ts index 7166a54b04..0ec1cc733f 100644 --- a/cli/test/sync_config_resolution.test.ts +++ b/cli/test/sync_config_resolution.test.ts @@ -30,7 +30,7 @@ Deno.test("Integration: wmill.yaml configuration produces expected results", asy await withContainerizedBackend(async (backend, tempDir) => { // Set up workspace profile with name "localhost_test" await setupWorkspaceProfile(backend); - + // Create wmill.yaml with settings await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: @@ -43,7 +43,7 @@ skipResources: true includeSettings: true includeSchedules: true includeTriggers: true`); - + // Test pull with wmill.yaml configuration const yamlResult = await backend.runCLICommand(['sync', 'pull', '--dry-run', '--json-output'], tempDir); if (yamlResult.code !== 0) { @@ -53,21 +53,21 @@ includeTriggers: true`); console.log("Stderr:", yamlResult.stderr); } assertEquals(yamlResult.code, 0); - + // Extract JSON from CLI output (skip log messages) const yamlData = parseJsonFromCLIOutput(yamlResult.stdout); - + // Should include settings.yaml due to includeSettings: true - const hasSettings = (yamlData.changes || []).some((change: any) => + const hasSettings = (yamlData.changes || []).some((change: any) => change.type === 'added' && change.path === 'settings.yaml' ); assertEquals(hasSettings, true); - + // Should NOT include resources or variables (due to skip flags) - const hasResources = (yamlData.changes || []).some((change: any) => + const hasResources = (yamlData.changes || []).some((change: any) => change.type === 'added' && change.path?.includes('.resource.yaml') ); - const hasVariables = (yamlData.changes || []).some((change: any) => + const hasVariables = (yamlData.changes || []).some((change: any) => change.type === 'added' && change.path?.includes('.variable.yaml') ); assertEquals(hasResources, false); @@ -79,35 +79,35 @@ Deno.test("Integration: settings.yaml inclusion respects includeSettings flag", await withContainerizedBackend(async (backend, tempDir) => { // Set up workspace profile with name "localhost_test" await setupWorkspaceProfile(backend); - - // Test 1: includeSettings: true should include settings.yaml + + // Test 1: includeSettings: true should include settings.yaml await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: - "**" includeSettings: true`); - + const includeResult = await backend.runCLICommand(['sync', 'pull', '--dry-run', '--json-output'], tempDir); assertEquals(includeResult.code, 0); - + // Extract JSON from CLI output (skip log messages) const includeData = parseJsonFromCLIOutput(includeResult.stdout); - const hasSettingsInclude = (includeData.changes || []).some((change: any) => + const hasSettingsInclude = (includeData.changes || []).some((change: any) => change.type === 'added' && change.path === 'settings.yaml' ); assertEquals(hasSettingsInclude, true); - + // Test 2: includeSettings: false should NOT include settings.yaml await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: - "**" includeSettings: false`); - + const excludeResult = await backend.runCLICommand(['sync', 'pull', '--dry-run', '--json-output'], tempDir); assertEquals(excludeResult.code, 0); - + // Extract JSON from CLI output (skip log messages) const excludeData = parseJsonFromCLIOutput(excludeResult.stdout); - const hasSettingsExclude = (excludeData.changes || []).some((change: any) => + const hasSettingsExclude = (excludeData.changes || []).some((change: any) => change.type === 'added' && change.path === 'settings.yaml' ); assertEquals(hasSettingsExclude, false); @@ -118,30 +118,86 @@ Deno.test("Integration: resource/variable filtering respects skip flags", async await withContainerizedBackend(async (backend, tempDir) => { // Set up workspace profile with name "localhost_test" await setupWorkspaceProfile(backend); - - // Test skipResources: true + + // Test skipResources: true await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun includes: - "**" skipResources: true skipVariables: false`); - + const result = await backend.runCLICommand(['sync', 'pull', '--dry-run', '--json-output'], tempDir); assertEquals(result.code, 0); - + // Extract JSON from CLI output (skip log messages) const data = parseJsonFromCLIOutput(result.stdout); - + // Should NOT include resources - const hasResources = (data.changes || []).some((change: any) => + const hasResources = (data.changes || []).some((change: any) => change.type === 'added' && change.path?.includes('.resource.yaml') ); assertEquals(hasResources, false); - + // Should include variables (not skipped) - const hasVariables = (data.changes || []).some((change: any) => + const hasVariables = (data.changes || []).some((change: any) => change.type === 'added' && change.path?.includes('.variable.yaml') ); assertEquals(hasVariables, true); }); -}); \ No newline at end of file +}); + +// ============================================================================= +// CLI FLAG OVERRIDE TESTS +// Tests for CLI flags overriding configuration file settings +// ============================================================================= + +Deno.test("CLI skip flags override wmill.yaml configuration", async () => { + await withContainerizedBackend(async (backend, tempDir) => { + // Set up workspace profile with name "localhost_test" + await setupWorkspaceProfile(backend); + + // Create wmill.yaml that INCLUDES resources by default (skipResources: false) + await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: + - f/** + - u/** +skipResources: false +skipResourceTypes: false +includeSettings: true`); + + // Test 1: Without CLI flags - should respect wmill.yaml (include resources) + const configResult = await backend.runCLICommand(['sync', 'pull', '--dry-run', '--json-output'], tempDir); + assertEquals(configResult.code, 0); + + const configData = parseJsonFromCLIOutput(configResult.stdout); + + + // Should include resources (not skipped by config) + const hasResources = (configData.changes || []).some((change: any) => + change.type === 'added' && change.path?.includes('.resource.yaml') + ); + assertEquals(hasResources, true, "Resources should be included by wmill.yaml config"); + + // Test 2: With CLI --skip-resources flag - should override wmill.yaml to skip resources + const overrideResult = await backend.runCLICommand([ + 'sync', 'pull', '--dry-run', '--json-output', + '--skip-resources', // CLI flag should override config to skip resources + '--skip-resource-types' // CLI flag should override config to skip resource types + ], tempDir); + assertEquals(overrideResult.code, 0); + + const overrideData = parseJsonFromCLIOutput(overrideResult.stdout); + + // Should NOT include resources (CLI flag overrides config) + const hasResourcesOverride = (overrideData.changes || []).some((change: any) => + change.type === 'added' && change.path?.includes('.resource.yaml') + ); + assertEquals(hasResourcesOverride, false, "CLI --skip-resources flag should override wmill.yaml to exclude resources"); + + // Should NOT include resource types (CLI flag overrides config) + const hasResourceTypesOverride = (overrideData.changes || []).some((change: any) => + change.type === 'added' && change.path?.includes('.resource-type.yaml') + ); + assertEquals(hasResourceTypesOverride, false, "CLI --skip-resource-types flag should override wmill.yaml to exclude resource types"); + }); +}); diff --git a/cli/trigger.ts b/cli/trigger.ts index 2edd6b1674..df21755794 100644 --- a/cli/trigger.ts +++ b/cli/trigger.ts @@ -16,9 +16,8 @@ import { parseFromFile, removeType, } from "./types.ts"; -import { requireLogin } from "./context.ts"; -import { validatePath } from "./context.ts"; -import { resolveWorkspace } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { validatePath, resolveWorkspace } from "./context.ts"; type Trigger = { http: HttpTrigger; diff --git a/cli/user.ts b/cli/user.ts index 70e069f303..668f294d67 100644 --- a/cli/user.ts +++ b/cli/user.ts @@ -1,5 +1,5 @@ // deno-lint-ignore-file no-explicit-any -import { requireLogin } from "./context.ts"; +import { requireLogin } from "./auth.ts"; import { GlobalOptions, isSuperset, diff --git a/cli/utils.ts b/cli/utils.ts index 1defd68b3f..af5548330c 100644 --- a/cli/utils.ts +++ b/cli/utils.ts @@ -2,7 +2,7 @@ // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-nocheck This file is copied from a JS project, so it's not type-safe. -import { log, encodeHex, SEP } from "./deps.ts"; +import { colors, log, encodeHex, SEP } from "./deps.ts"; import crypto from "node:crypto"; export function deepEqual(a: T, b: T): boolean { @@ -167,7 +167,7 @@ export async function selectRepository( if (repositories.length === 1) { const repoPath = repositories[0].git_repo_resource_path.replace(/^\$res:/, ""); - log.info(`Using repository: ${repoPath}`); + log.info(colors.cyan(`Auto-selected repository: ${colors.bold(repoPath)}`)); return repositories[0]; } @@ -196,4 +196,4 @@ export async function selectRepository( }); return repositories.find((r) => r.git_repo_resource_path === selectedRepo)!; -} \ No newline at end of file +} diff --git a/cli/variable.ts b/cli/variable.ts index 789334c44c..8524959c32 100644 --- a/cli/variable.ts +++ b/cli/variable.ts @@ -1,5 +1,6 @@ // deno-lint-ignore-file no-explicit-any -import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { requireLogin } from "./auth.ts"; +import { resolveWorkspace, validatePath } from "./context.ts"; import { GlobalOptions, isSuperset, diff --git a/cli/workspace.ts b/cli/workspace.ts index b2af05d089..9c0b865a00 100644 --- a/cli/workspace.ts +++ b/cli/workspace.ts @@ -3,9 +3,9 @@ import { GlobalOptions } from "./types.ts"; import { getRootStore } from "./store.ts"; import { loginInteractive, tryGetLoginInfo } from "./login.ts"; import { colors, Command, Confirm, Input, log, setClient, Table } from "./deps.ts"; +import { requireLogin } from "./auth.ts"; import * as wmill from "./gen/services.gen.ts"; -import { requireLogin } from "./context.ts"; export interface Workspace { remote: string; @@ -258,11 +258,11 @@ export async function add( export async function addWorkspace(workspace: Workspace, opts: any) { workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases! - + // Check for conflicts before adding const existingWorkspaces = await allWorkspaces(opts.configDir); const isInteractive = Deno.stdin.isTerminal() && Deno.stdout.isTerminal() && !opts.force; - + // Check 1: Workspace name already exists const nameConflict = existingWorkspaces.find(w => w.name === workspace.name); if (nameConflict) { @@ -274,7 +274,7 @@ export async function addWorkspace(workspace: Workspace, opts: any) { log.info(colors.red.bold(`❌ Workspace name "${workspace.name}" already exists!`)); log.info(` Existing: ${nameConflict.workspaceId} on ${nameConflict.remote}`); log.info(` New: ${workspace.workspaceId} on ${workspace.remote}`); - + if (!isInteractive) { // In non-interactive mode (tests, scripts), auto-overwrite with force flag if (opts.force) { @@ -287,7 +287,7 @@ export async function addWorkspace(workspace: Workspace, opts: any) { message: "Do you want to overwrite the existing workspace?", default: false, }); - + if (!overwrite) { log.info(colors.yellow("Operation cancelled.")); return; @@ -295,20 +295,20 @@ export async function addWorkspace(workspace: Workspace, opts: any) { } } } - + // Check 2: Same (remote, workspaceId) tuple already exists under different name - const tupleConflict = existingWorkspaces.find(w => - w.remote === workspace.remote && - w.workspaceId === workspace.workspaceId && + const tupleConflict = existingWorkspaces.find(w => + w.remote === workspace.remote && + w.workspaceId === workspace.workspaceId && w.name !== workspace.name ); - + if (tupleConflict) { log.info(colors.red.bold(`❌ Workspace ${workspace.workspaceId} on ${workspace.remote} already exists!`)); log.info(` Existing name: "${tupleConflict.name}"`); log.info(` New name: "${workspace.name}"`); log.info(colors.yellow(`\nNote: Backend constraint prevents duplicate (remote, workspaceId) combinations.`)); - + if (!isInteractive) { // In non-interactive mode (tests, scripts), auto-overwrite with force flag if (opts.force) { @@ -321,20 +321,20 @@ export async function addWorkspace(workspace: Workspace, opts: any) { message: `Do you want to overwrite the existing workspace "${tupleConflict.name}"?`, default: false, }); - + if (!overwrite) { log.info(colors.yellow("Operation cancelled.")); return; } } - + // Remove the conflicting workspace await removeWorkspace(tupleConflict.name, true, opts); } - + // Remove existing workspace with same name (if updating) await removeWorkspace(workspace.name, true, opts); - + // Add the new workspace const file = await Deno.open((await getRootStore(opts.configDir)) + "remotes.ndjson", { append: true, diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 3be8c6c5b8..ca749b02f0 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index f3f1e26cee..4b001f5d10 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/flake.nix b/flake.nix index 34bad7cef7..0a8f3b9138 100644 --- a/flake.nix +++ b/flake.nix @@ -211,6 +211,7 @@ DENO_PATH = "${pkgs.deno}/bin/deno"; GO_PATH = "${pkgs.go}/bin/go"; PHP_PATH = "${pkgs.php}/bin/php"; + COMPOSER_PATH = "${pkgs.php84Packages.composer}/bin/composer"; BUN_PATH = "${pkgs.bun}/bin/bun"; UV_PATH = "${pkgs.uv}/bin/uv"; NU_PATH = "${pkgs.nushell}/bin/nu"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4ff437e93f..7f51ee668b 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.507.1", + "version": "1.508.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.507.1", + "version": "1.508.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 46d07e224e..f2b85a0b72 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.507.1", + "version": "1.508.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index b7c274a0b4..1a634b42e2 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -648,6 +648,16 @@ reorderable /> + {:else if enum_ && (Array.isArray(value) || value == undefined)} +
+ dispatch('focus')} + reorderable + /> +
{:else if itemsType?.enum != undefined && Array.isArray(itemsType?.enum) && (Array.isArray(value) || value == undefined)}
@@ -894,6 +905,7 @@ {onlyMaskPassword} {disablePortal} {disabled} + {prettifyHeader} bind:schema={ () => ({ properties: obj.properties ?? {}, @@ -924,6 +936,7 @@ {onlyMaskPassword} {disablePortal} {disabled} + {prettifyHeader} hiddenArgs={['label', 'kind']} schema={{ properties: obj.properties, @@ -999,6 +1012,7 @@ {onlyMaskPassword} {disablePortal} {disabled} + {prettifyHeader} bind:schema={ () => ({ properties, @@ -1034,6 +1048,7 @@ {onlyMaskPassword} {disablePortal} {disabled} + {prettifyHeader} schema={{ properties, order, diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index b96a8a44bd..f7972a0055 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -38,6 +38,7 @@ import type { DisplayResultUi } from './custom_ui' import { getContext, hasContext, createEventDispatcher, onDestroy } from 'svelte' import { toJsonStr } from '$lib/utils' + import { userStore } from '$lib/stores' const IMG_MAX_SIZE = 10000000 const TABLE_MAX_SIZE = 5000000 @@ -779,34 +780,38 @@ language={json} code={toJsonStr(result).replace(/\\n/g, '\n')} /> - + {#if $userStore} + + {/if} {:else if !result?.disable_download} - + {#if $userStore} + + {/if} {/if}
{#if typeof result?.s3 === 'string'} diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 41dc59c0f2..749f8d163d 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -593,10 +593,10 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS let t = { postgresql: 'postgres', mysql: 'mysql', bigquery: 'bigquery' }[resType] if (!t) { sendUserToast(`Resource type ${resType} is not supported in DuckDB`, true) - editor.insertAtCursor(`'$res:${path}'`) + editor.insertAtCursor(`'res://${path}'`) return } else { - editor.insertAtCursor(`ATTACH '$res:${path}' AS db (TYPE ${t});`) + editor.insertAtCursor(`ATTACH 'res://${path}' AS db (TYPE ${t});`) } } diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 2672b491de..da9fb506ca 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -86,6 +86,7 @@ } from './stepHistoryLoader.svelte' import type { FlowBuilderProps } from './flow_builder' import { ModulesTestStates } from './modulesTest.svelte' + import FlowAssetsHandler, { initFlowGraphAssetsCtx } from './flows/FlowAssetsHandler.svelte' let { initialPath = $bindable(''), @@ -609,6 +610,11 @@ outputPickerOpenFns }) + setContext( + 'FlowGraphAssetContext', + initFlowGraphAssetsCtx({ getModules: () => flowStore.val.value.modules }) + ) + // Add triggers context store const triggersState = $state( new Triggers( @@ -1256,3 +1262,10 @@ renderCount += 1 }} /> + + diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index d03dc4fe03..5454879028 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -122,7 +122,7 @@ args: Record, restartedFrom: RestartedFrom | undefined ) { - if (stepHistoryLoader?.flowJobInitial) { + if (stepHistoryLoader?.flowJobInitial !== false) { stepHistoryLoader?.setFlowJobInitial(false) } try { diff --git a/frontend/src/lib/components/FlowPreviewResult.svelte b/frontend/src/lib/components/FlowPreviewResult.svelte index d92192fadb..ceaebe5b15 100644 --- a/frontend/src/lib/components/FlowPreviewResult.svelte +++ b/frontend/src/lib/components/FlowPreviewResult.svelte @@ -1,5 +1,5 @@ {#if notAnonynmous} @@ -906,7 +950,7 @@

For performance reasons, only the last 20 items are shown by default + {#if $userStore} + + {/if} diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 2983d4b35b..78282eca09 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -89,15 +89,16 @@ class AIChatManager { open = $derived(chatState.size > 0) - constructor() { - loadApiTools() - .then((tools) => { - this.apiTools = tools - }) - .catch((err) => { - console.error('Error loading api tools', err) - this.apiTools = [] - }) + loadApiTools = async () => { + try { + this.apiTools = await loadApiTools() + if (this.mode === AIMode.NAVIGATOR) { + this.tools = [this.changeModeTool, ...navigatorTools, ...this.apiTools] + } + } catch (err) { + console.error('Error loading api tools', err) + this.apiTools = [] + } } setAiChatInput(aiChatInput: AIChatInput | null) { @@ -604,6 +605,10 @@ class AIChatManager { } } + if (this.mode === AIMode.NAVIGATOR && this.apiTools.length === 0) { + await this.loadApiTools() + } + await this.chatRequest({ ...params }) diff --git a/frontend/src/lib/components/copilot/chat/navigator/apiTools.ts b/frontend/src/lib/components/copilot/chat/navigator/apiTools.ts index f03920415f..1aea113ba6 100644 --- a/frontend/src/lib/components/copilot/chat/navigator/apiTools.ts +++ b/frontend/src/lib/components/copilot/chat/navigator/apiTools.ts @@ -1,7 +1,7 @@ import type { ChatCompletionTool } from 'openai/resources/index.mjs' import type { Tool } from '../shared' import { get } from 'svelte/store' -import { workspaceStore } from '$lib/stores' +import { workspaceStore, enterpriseLicense } from '$lib/stores' // OpenAPI type definitions interface OpenAPIParameter { @@ -54,6 +54,88 @@ interface OpenAPISpec { paths: { [path: string]: OpenAPIPathItem } + components?: { + parameters?: { + [name: string]: OpenAPIParameter + } + schemas?: { + [name: string]: any + } + } +} + +interface OpenAPIParameterWithRef { + $ref?: string + name?: string + in?: string + description?: string + required?: boolean + schema?: { + type?: string + format?: string + } +} + +/** + * Dereferences parameter $ref references in an OpenAPI spec + * Only resolves parameter references, not schemas or other components + */ +function dereferenceParameters(spec: OpenAPISpec): OpenAPISpec { + if (!spec.components?.parameters) { + return spec + } + + const resolveParameterRef = (paramRef: OpenAPIParameterWithRef): OpenAPIParameter => { + if (paramRef.$ref) { + // Extract parameter name from $ref (e.g., "#/components/parameters/WorkspaceId" -> "WorkspaceId") + const refPath = paramRef.$ref.split('/') + if (refPath.length >= 4 && refPath[1] === 'components' && refPath[2] === 'parameters') { + const paramName = refPath[3] + const resolvedParam = spec.components?.parameters?.[paramName] + if (resolvedParam) { + return resolvedParam + } + } + console.warn(`Could not resolve parameter reference: ${paramRef.$ref}`) + return paramRef as OpenAPIParameter + } + return paramRef as OpenAPIParameter + } + + const processParameters = (parameters: OpenAPIParameterWithRef[]): OpenAPIParameter[] => { + return parameters.map(resolveParameterRef) + } + + const dereferencedSpec: OpenAPISpec = { + ...spec, + paths: {} + } + + // Process each path + for (const [pathKey, pathItem] of Object.entries(spec.paths)) { + const newPathItem: OpenAPIPathItem = { ...pathItem } + + // Dereference path-level parameters + if (pathItem.parameters) { + newPathItem.parameters = processParameters(pathItem.parameters as OpenAPIParameterWithRef[]) + } + + // Dereference operation-level parameters + const methods = ['get', 'post', 'put', 'delete', 'patch', 'options'] as const + for (const method of methods) { + const operation = pathItem[method] + if (operation?.parameters) { + newPathItem[method] = { + ...operation, + parameters: processParameters(operation.parameters as OpenAPIParameterWithRef[]) + } + } + } + + dereferencedSpec.paths[pathKey] = newPathItem + } + + return dereferencedSpec } const buildApiCallTools = ( @@ -124,6 +206,10 @@ export function buildToolsFromOpenApi( // Add path parameters for (const param of pathParams) { + if (param.name === 'workspace') { + continue + } + parameters.properties[param.name] = { type: param.schema?.type || 'string', description: param.description || `Path parameter: ${param.name}` @@ -167,13 +253,13 @@ export function buildToolsFromOpenApi( } const tool = buildApiCallTools( - 'api_' + op.operationId, + 'api_' + op.operationId.replace(/\s+/g, ''), op.summary || op.description || `${method.toUpperCase()} ${path}`, parameters ) // Store the endpoint path in the map - endpointMap['api_' + op.operationId] = `${method.toUpperCase()} ${path}` + endpointMap['api_' + op.operationId.replace(/\s+/g, '')] = `${method.toUpperCase()} ${path}` tools.push(tool) } @@ -193,7 +279,6 @@ export function createApiTools( const toolName = chatTool.function.name let endpoint = endpointMap[toolName] || '' endpoint = endpoint.replace('{workspace}', get(workspaceStore) as string) - toolCallbacks.setToolStatus(toolId, `Calling API endpoint (${endpoint})...`) try { // Extract method and path from endpoint @@ -212,8 +297,8 @@ export function createApiTools( if (key === 'body') continue // Body is handled separately // Check if this is a path parameter - if (url.includes(`:${key}`)) { - url = url.replace(`:${key}`, encodeURIComponent(String(value))) + if (url.includes(`{${key}}`)) { + url = url.replace(`{${key}}`, encodeURIComponent(String(value))) } else { // Assume it's a query parameter queryParams[key] = String(value) @@ -231,18 +316,32 @@ export function createApiTools( // Log the constructed URL console.log(`Calling API: ${method} ${url} with args: ${JSON.stringify(args)}`) + toolCallbacks.setToolStatus(toolId, `Calling API endpoint (${url})...`) + const response = await fetch(url, { method: method }) - const data = await response.json() - // For now, return a placeholder response - // In a real implementation, we would make the actual API call here - toolCallbacks.setToolStatus(toolId, `API call to ${endpoint} completed`) - return JSON.stringify({ - success: true, - data: data - }) + if (response.ok) { + let result = '' + if (response.headers.get('content-type')?.includes('application/json')) { + result = await response.json() + } else { + result = await response.text() + } + toolCallbacks.setToolStatus(toolId, `API call to ${url} completed`) + return JSON.stringify({ + success: true, + data: result + }) + } else { + const text = await response.text() + toolCallbacks.setToolStatus(toolId, `API call to ${url} failed`) + return JSON.stringify({ + success: false, + data: text + }) + } } catch (error) { toolCallbacks.setToolStatus(toolId, `API call to ${endpoint} failed`) console.error(`Error calling API to ${endpoint}:`, error) @@ -256,9 +355,14 @@ export function createApiTools( export async function loadApiTools(): Promise[]> { try { const response = await fetch('/api/openapi.json') - const openApiSpec = (await response.json()) as OpenAPISpec + const rawOpenApiSpec = (await response.json()) as OpenAPISpec + + // Dereference parameter references + const openApiSpec = dereferenceParameters(rawOpenApiSpec) + const pathsToInclude = [ 'jobs', + 'jobs_u', 'scripts', 'flows', 'resources', @@ -267,6 +371,14 @@ export async function loadApiTools(): Promise[]> { 'workers' ] + // call srch endpoint to check if it's available + if (get(enterpriseLicense)) { + const srchResponse = await fetch(`/api/srch/index/search/enabled`) + if (srchResponse.ok) { + pathsToInclude.push('srch/w') // job search + } + } + const { tools: apiTools, endpointMap } = buildToolsFromOpenApi(openApiSpec, { pathFilter: (path) => pathsToInclude.some((p) => path.includes(`/${p}/`)), methodFilter: ['get'] diff --git a/frontend/src/lib/components/copilot/chat/navigator/core.ts b/frontend/src/lib/components/copilot/chat/navigator/core.ts index aecf60bc30..9194ec290d 100644 --- a/frontend/src/lib/components/copilot/chat/navigator/core.ts +++ b/frontend/src/lib/components/copilot/chat/navigator/core.ts @@ -5,7 +5,7 @@ import type { } from 'openai/resources/index.mjs' import type { Tool } from '../shared' import { ResourceService } from '$lib/gen' -import { workspaceStore } from '$lib/stores' +import { enterpriseLicense, workspaceStore } from '$lib/stores' import { get } from 'svelte/store' import { triggerablesByAi } from '../sharedChatState.svelte' @@ -29,6 +29,7 @@ INSTRUCTIONS: - If you are asked to fill a form or act on an input, input the existing json object and change the fields the user asked you to change. Take into account the prompt_for_ai field of the schema to know what and how to do changes. Then tell the user that you have updated the form, and ask him to review the changes before running the script or flow. - For form inputs where format starts with "resource-" and is not "resource-obj", fetch the available resources using get_available_resources, and then use the resource_path prefixed with "$res:" to fill the input. - If you are not sure about an input, set the ones you are sure about, and then ask the user for the value of the input you are not sure about. +${get(enterpriseLicense) ? `- If asked to look through the jobs logs, use the /srch/w/{workspace}/index/search/job endpoint to search for the relevant jobs runs. Then use /w/{workspace}/jobs_u/get to get the logs of each job.` : ''} GENERAL PRINCIPLES: - Be concise but thorough diff --git a/frontend/src/lib/components/flows/FlowAssetsHandler.svelte b/frontend/src/lib/components/flows/FlowAssetsHandler.svelte new file mode 100644 index 0000000000..cd4ebc98c4 --- /dev/null +++ b/frontend/src/lib/components/flows/FlowAssetsHandler.svelte @@ -0,0 +1,162 @@ + + + + +{#if enableParser} + {#each allModules as mod (mod.id)} + {#if mod.value.type === 'rawscript'} + {@const v = mod.value} + parseAndUpdateRawScriptModule(v)} /> + {/if} + {/each} +{/if} + +{#if flowGraphAssetsCtx} + + + +{/if} diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index 58dd9ae29a..48fec0db83 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -51,10 +51,9 @@ import { workspaceStore } from '$lib/stores' import { checkIfParentLoop } from '../utils' import ModulePreviewResultViewer from '$lib/components/ModulePreviewResultViewer.svelte' - import { refreshStateStore, usePromise } from '$lib/svelte5Utils.svelte' + import { refreshStateStore } from '$lib/svelte5Utils.svelte' import { getStepHistoryLoaderContext } from '$lib/components/stepHistoryLoader.svelte' import AssetsDropdownButton from '$lib/components/assets/AssetsDropdownButton.svelte' - import { inferAssets } from '$lib/infer' const { selectedId, @@ -123,6 +122,8 @@ let testIsLoading = $state(false) let scriptProgress = $state(undefined) + let assets = $derived((flowModule.value.type === 'rawscript' && flowModule.value.assets) || []) + function onModulesChange(savedModule: FlowModule | undefined, flowModule: FlowModule) { // console.log('onModulesChange', savedModule, flowModule) return savedModule?.value?.type === 'rawscript' && @@ -299,19 +300,6 @@ } }) - let assets = usePromise( - async () => - flowModule.value.type === 'rawscript' - ? await inferAssets(flowModule.value.language, flowModule.value.content) - : undefined, - { clearValueOnRefresh: false, loadInit: false } - ) - $effect(() => { - if (flowModule.value.type !== 'rawscript') return - ;[flowModule.value.content, flowModule.value.language] - untrack(() => assets.refresh()) - }) - let rawScriptLang = $derived( flowModule.value.type == 'rawscript' ? flowModule.value.language : undefined ) @@ -422,11 +410,8 @@ {#if !noEditor} {#key flowModule.id}

- {#if assets.value?.length} - + {#if assets?.length} + {/if}
// Maps module ids to their assets s3FilePicker: S3FilePicker | undefined dbManagerDrawer: DbManagerDrawer | undefined resourceEditorDrawer: ResourceEditorDrawer | undefined // Maps resource paths to their metadata. undefined is for error resourceMetadataCache: Record + additionalAssetsMap: Record + computeAssetsCount: (asset: Asset) => number }> diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index ef4bcef38b..bfdeca1e21 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -1,12 +1,5 @@ @@ -249,8 +245,10 @@ 'bg-surface h-6 flex items-center gap-1.5 rounded-sm text-tertiary border overflow-clip', isSelected ? 'bg-surface-secondary !border-surface-inverse' : 'border-transparent' )} - onmouseenter={() => (flowGraphAssetsCtx.val.selectedAsset = data.asset)} - onmouseleave={() => (flowGraphAssetsCtx.val.selectedAsset = undefined)} + onmouseenter={() => + flowGraphAssetsCtx && (flowGraphAssetsCtx.val.selectedAsset = data.asset)} + onmouseleave={() => + flowGraphAssetsCtx && (flowGraphAssetsCtx.val.selectedAsset = undefined)} > {/if} - Used in {pluralize(usageCount, 'step')}
+ Used in {pluralize( + flowGraphAssetsCtx?.val.computeAssetsCount?.(data.asset) ?? -1, + 'step' + )}
{ if (data.asset.kind === 'resource') - flowGraphAssetsCtx.val.resourceEditorDrawer?.initEdit(data.asset.path) + flowGraphAssetsCtx?.val.resourceEditorDrawer?.initEdit(data.asset.path) }} > {data.asset.path} @@ -298,6 +299,8 @@ {formatAssetKind({ ...data.asset, metadata: cachedResourceMetadata })} +
+ fdsfs {/snippet} diff --git a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte index 994983956d..88fb4b1059 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte @@ -14,12 +14,12 @@ data: AssetsOverflowedN['data'] } let { data }: Props = $props() - const flowGraphAssetsCtx = getContext('FlowGraphAssetContext') + const flowGraphAssetsCtx = getContext('FlowGraphAssetContext') let isOpen = $state(false) let includesSelected = $derived( - data.overflowedAssets.some((asset) => assetEq(flowGraphAssetsCtx.val.selectedAsset, asset)) + data.overflowedAssets.some((asset) => assetEq(flowGraphAssetsCtx?.val.selectedAsset, asset)) ) let wasOpenedBecauseOfExternalSelected = false diff --git a/frontend/src/lib/components/icons/AssetGenericIcon.svelte b/frontend/src/lib/components/icons/AssetGenericIcon.svelte index 5be0c0c438..be8f4feb58 100644 --- a/frontend/src/lib/components/icons/AssetGenericIcon.svelte +++ b/frontend/src/lib/components/icons/AssetGenericIcon.svelte @@ -3,7 +3,6 @@ import type { AssetKind } from '../assets/lib' import AssetResIcon from './AssetResIcon.svelte' import AssetS3Icon from './AssetS3Icon.svelte' - import AssetVarIcon from './AssetVarIcon.svelte' interface Props { size?: string @@ -19,8 +18,6 @@ {:else if assetKind == 'resource'} -{:else if assetKind == 'variable'} - {:else} {/if} diff --git a/frontend/src/lib/components/icons/AssetVarIcon.svelte b/frontend/src/lib/components/icons/AssetVarIcon.svelte deleted file mode 100644 index be24443034..0000000000 --- a/frontend/src/lib/components/icons/AssetVarIcon.svelte +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - diff --git a/frontend/src/lib/components/runs/BatchReRunOptionsPane.svelte b/frontend/src/lib/components/runs/BatchReRunOptionsPane.svelte index 1c3eed8e60..6095f086c1 100644 --- a/frontend/src/lib/components/runs/BatchReRunOptionsPane.svelte +++ b/frontend/src/lib/components/runs/BatchReRunOptionsPane.svelte @@ -28,6 +28,7 @@ } from '$lib/components/jobs/batchReruns' import Toggle from '../Toggle.svelte' import { TriangleAlert } from 'lucide-svelte' + import { readFieldsRecursively } from '$lib/utils' let { selectedIds, @@ -149,7 +150,10 @@ (options[selected.kind][selected.script_path]?.use_latest_version ?? false)) ) - const jobGroupsPromise = $derived(selectedIds && untrack(() => fetchJobGroups())) + const jobGroupsPromise = $derived.by(() => { + readFieldsRecursively(selectedIds) + return untrack(() => fetchJobGroups()) + })
diff --git a/frontend/src/lib/components/runs/JobPreview.svelte b/frontend/src/lib/components/runs/JobPreview.svelte index 5ad5bc3dfa..19f4273681 100644 --- a/frontend/src/lib/components/runs/JobPreview.svelte +++ b/frontend/src/lib/components/runs/JobPreview.svelte @@ -15,8 +15,10 @@ import WorkflowTimeline from '../WorkflowTimeline.svelte' import Popover from '../Popover.svelte' import { isFlowPreview, isScriptPreview, truncateRev } from '$lib/utils' - import { createEventDispatcher, untrack } from 'svelte' + import { createEventDispatcher, setContext, untrack } from 'svelte' import { ListFilter } from 'lucide-svelte' + import FlowAssetsHandler, { initFlowGraphAssetsCtx } from '../flows/FlowAssetsHandler.svelte' + import JobAssetsViewer from '../assets/JobAssetsViewer.svelte' interface Props { id: string @@ -45,6 +47,11 @@ let viewTab = $state('result') + setContext( + 'FlowGraphAssetContext', + initFlowGraphAssetsCtx({ getModules: () => job?.raw_flow?.modules ?? [] }) + ) + function asWorkflowStatus(x: any): Record { return x as Record } @@ -193,6 +200,7 @@ Result Logs + Assets {#if isScriptPreview(job?.job_kind)} Code {/if} @@ -206,6 +214,8 @@
+ {:else if viewTab == 'assets'} + {:else}
{#if viewTab == 'logs'} @@ -264,3 +274,8 @@
{/if} + diff --git a/frontend/src/lib/components/script_builder.ts b/frontend/src/lib/components/script_builder.ts index 6ce7377846..dd5ac52fad 100644 --- a/frontend/src/lib/components/script_builder.ts +++ b/frontend/src/lib/components/script_builder.ts @@ -1,5 +1,5 @@ import type { NewScript } from '$lib/gen' -import type { AssetWithAccessType } from './assets/lib' +import type { AssetWithAltAccessType } from './assets/lib' import type { ScriptBuilderWhitelabelCustomUi } from './custom_ui' import type { DiffDrawerI } from './diff_drawer' import type { ScriptBuilderFunctionExports } from './scriptBuilder' @@ -9,7 +9,7 @@ import type { NewScriptWithDraftAndDraftTriggers, Trigger } from './triggers/uti export interface ScriptBuilderProps { script: NewScript & { draft_triggers?: Trigger[] - fallback_access_types?: AssetWithAccessType[] + assets?: AssetWithAltAccessType[] } disableAi?: boolean fullyLoaded?: boolean diff --git a/frontend/src/lib/components/settings/CreateToken.svelte b/frontend/src/lib/components/settings/CreateToken.svelte index fe9287ad52..d4ef7c0b44 100644 --- a/frontend/src/lib/components/settings/CreateToken.svelte +++ b/frontend/src/lib/components/settings/CreateToken.svelte @@ -5,11 +5,12 @@ import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' import Toggle from '../Toggle.svelte' - import { UserService, type NewToken } from '$lib/gen' - import { copyToClipboard } from '$lib/utils' - import { Clipboard } from 'lucide-svelte' - import ClipboardPanel from '../details/ClipboardPanel.svelte' + import { IntegrationService, UserService, type NewToken } from '$lib/gen' import { createEventDispatcher } from 'svelte' + import MultiSelect from '../select/MultiSelect.svelte' + import { safeSelectItems } from '../select/utils.svelte' + import TokenDisplay from './TokenDisplay.svelte' + import ScopeSelector from './ScopeSelector.svelte' interface Props { showMcpMode?: boolean @@ -17,7 +18,7 @@ newTokenLabel?: string defaultNewTokenWorkspace?: string scopes?: string[] - actions?: () => void + onTokenCreated?: (token: string) => void displayCreateToken?: boolean } @@ -25,7 +26,7 @@ showMcpMode = false, defaultNewTokenWorkspace, scopes, - actions, + onTokenCreated, newTokenLabel = $bindable(undefined) }: Props = $props() @@ -38,12 +39,13 @@ let newMcpScope = $state('favorites') let loadingApps = $state(false) let errorFetchApps = $state(false) - // let allApps = $state([]) + let allApps = $state([]) + + let customScopes = $state([]) + let showCustomScopes = $state(false) - function handleCopyClick() { - copyToClipboard(newToken ?? '') - } const dispatch = createEventDispatcher() + function ensureCurrentWorkspaceIncluded( workspacesList: UserWorkspace[], currentWorkspace: string | undefined @@ -71,6 +73,8 @@ if (newMcpApps.length > 0) { tokenScopes.push(`mcp:hub:${newMcpApps.join(',')}`) } + } else if (showCustomScopes && customScopes.length > 0) { + tokenScopes = customScopes } const createdToken = await UserService.createToken({ @@ -89,8 +93,8 @@ } dispatch('tokenCreated', newToken ?? newMcpToken) - if (actions) { - actions() + if (onTokenCreated) { + onTokenCreated(newToken ?? newMcpToken ?? '') } mcpCreationMode = false } catch (err) { @@ -100,37 +104,37 @@ const workspaces = $derived(ensureCurrentWorkspaceIncluded($userWorkspaces, $workspaceStore)) const mcpBaseUrl = $derived(`${window.location.origin}/api/mcp/w/${newTokenWorkspace}/sse?token=`) + + $effect(() => { + if (mcpCreationMode) { + getAllApps() + } else { + newMcpApps = [] + } + }) + + async function getAllApps() { + if (allApps.length > 0) { + return + } + try { + loadingApps = true + allApps = ( + await IntegrationService.listHubIntegrations({ + kind: 'script' + }) + ).map((x) => x.name) + } catch (err) { + console.error('Hub is not available') + allApps = [] + errorFetchApps = true + } finally { + loadingApps = false + } + }
- {#if newToken} -
-
- Added token: -
-
- Make sure to copy your personal access token now. You won't be able to see it again! -
-
- {/if} - - {#if newMcpToken} -
-

New MCP URL:

- -

- Make sure to copy this URL now. You won't be able to see it again! -

-
- {/if} -

Add a new token

@@ -176,7 +180,27 @@
{/if} -
+ {#if !mcpCreationMode && (!scopes || scopes.length === 0)} +
+ { + showCustomScopes = e.detail + }} + options={{ + right: 'Limit token permissions', + rightTooltip: + 'By default, tokens have full API access. Enable this to restrict the token to specific scopes.' + }} + size="xs" + /> + {#if showCustomScopes} + + {/if} +
+ {/if} + +
{#if mcpCreationMode}
Scope @@ -205,7 +229,12 @@ {:else if errorFetchApps}
Error fetching apps
{:else} - + {/if}
@@ -259,4 +288,12 @@
+ + {#if newToken} + + {/if} + + {#if newMcpToken} + + {/if}
diff --git a/frontend/src/lib/components/settings/ScopeSelector.svelte b/frontend/src/lib/components/settings/ScopeSelector.svelte new file mode 100644 index 0000000000..329317518c --- /dev/null +++ b/frontend/src/lib/components/settings/ScopeSelector.svelte @@ -0,0 +1,739 @@ + + +
+ {#if loading} +
+ +
+ {:else if error} +
+

{error}

+ +
+ {:else if scopeDomains} +
+
+

+ Selected Scopes ({selectedScopes.length}) +

+ +
+ + {#if selectedScopes.length === 0} +

No scopes selected. Token will have full access.

+ {:else if hasAdministratorScope} +

Administrator scope grants full access to all resources.

+ {:else} +
+ {#each selectedScopes.slice(0, 10) as scope} + + {scope} + + + {/each} + {#if selectedScopes.length > 10} + + +{selectedScopes.length - 10} more + + {/if} +
+ {/if} +
+ +
+ {#each scopeDomains as domain} + {@const domainState = getDomainState(domain.name)} + {@const isExpanded = domainState?.isExpanded || false} + {@const isDomainSelected = domainState?.hasFullAccess || false} + {@const selectedScopes = getSelectedScopesForDomain(domain)} + +
+ + +
toggleDomainExpansion(domain.name)} + > +
+
+ +
+ +
+ handleDomainCheckboxChange(domain, e.currentTarget.checked)} + onclick={(e) => e.stopPropagation()} + class="!w-4 !h-4 cursor-pointer" + /> +
+ +
+
+ + + {#each selectedScopes as scope} + + {scope} + + + {/each} +
+ {#if domain.description} +

{domain.description}

+ {/if} +
+
+
+ + {#if isExpanded} +
+
+ {#each domain.scopes as scope} + {@const scopeState = domainState?.scopes[scope.value]} + {@const isSelected = scopeState?.isSelected || false} + {@const resourcePathArray = scopeState?.resourcePaths || []} + {@const currentInput = scopeState?.currentInputValue || ''} + {@const pathError = scopeState?.pathError} + {@const isDisabled = disabled || isScopeDisabled(scope, domain)} + +
+
+ +
+ {#if scope.requires_resource_path} + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
+
+ { + if (scopeState) { + scopeState.currentInputValue = e.currentTarget.value + scopeState.pathError = undefined + } + }} + placeholder="e.g. f/folder/*, u/user/path" + onkeydown={(e) => { + if (e.key === 'Enter' && currentInput.trim()) { + e.preventDefault() + addResourcePath(scope.value, currentInput) + } + }} + class="flex-1 text-sm px-3 py-2 border rounded-lg focus:ring-2 focus:ring-blue-500 focus:border-blue-500 bg-surface" + /> + +
+ {#if pathError} +

{pathError}

+ {/if} +
+ {/snippet} + + {/if} +
+
+ + {#if scope.requires_resource_path && resourcePathArray.length > 0} +
+ {#each resourcePathArray as path} + + {path} + + + {/each} +
+ {/if} +
+ {/each} +
+
+ {/if} +
+ {/each} +
+ {/if} +
diff --git a/frontend/src/lib/components/settings/TokenDisplay.svelte b/frontend/src/lib/components/settings/TokenDisplay.svelte new file mode 100644 index 0000000000..cc48b1d2b9 --- /dev/null +++ b/frontend/src/lib/components/settings/TokenDisplay.svelte @@ -0,0 +1,124 @@ + + +
+
+
+ {#if type === 'mcp'} + + + + {:else} + + + + {/if} +
+
+

+ {displayTitle} +

+ + {#if type === 'mcp' && mcpUrl} +
+
+ + + +
+
+ + +
+
+ + {token} + + +
+
+
+ + Make sure to copy both the URL and token now. You won't be able to see them again! + +
+

+ Next steps: Use this URL in your MCP-compatible client (like Claude Desktop) to access your Windmill scripts and flows as tools. +

+
+
+ {:else} +
+
+ + {token} + + +
+
+
+ + Make sure to copy your personal access token now. You won't be able to see it again! + +
+ {/if} +
+
+
\ No newline at end of file diff --git a/frontend/src/lib/components/settings/TokensTable.svelte b/frontend/src/lib/components/settings/TokensTable.svelte index 1fa49cc96b..2056def624 100644 --- a/frontend/src/lib/components/settings/TokensTable.svelte +++ b/frontend/src/lib/components/settings/TokensTable.svelte @@ -1,19 +1,10 @@ -
-

Tokens

-
- +
+
+ Authenticate to the Windmill API with access tokens.
-
-
- Authenticate to the Windmill API with access tokens. -
-
- {#if newToken} -
-
- Added token: -
-
- Make sure to copy your personal access token now. You won't be able to see it again! -
-
- {/if} - - {#if newMcpToken} -
-

New MCP URL:

- -

- Make sure to copy this URL now. You won't be able to see it again! -

-
- {/if} - - {#if displayCreateToken} -
-

Add a new token

- - {#if showMcpMode} -
- { - mcpCreationMode = e.detail - if (e.detail) { - newTokenLabel = 'MCP token' - newTokenExpiration = undefined - newTokenWorkspace = $workspaceStore - } else { - newTokenLabel = undefined - newTokenExpiration = undefined - newTokenWorkspace = defaultNewTokenWorkspace - } - }} - checked={mcpCreationMode} - options={{ - right: 'Generate MCP URL', - rightTooltip: - 'Generate a new MCP URL to make your scripts and flows available as tools through your LLM clients.', - rightDocumentationLink: 'https://www.windmill.dev/docs/core_concepts/mcp' - }} - size="xs" - aiId="account-settings-create-mcp-token" - aiDescription="Create a new MCP token to authenticate to the Windmill API" - /> -
- {/if} - - {#if scopes != undefined} -
- Scope - {#each scopes as scope} - - {/each} -
- {/if} - -
- {#if mcpCreationMode} -
- Scope - - {#snippet children({ item })} - - - {/snippet} - -
- -
- Hub scripts (optional) - {#if loadingApps} -
Loading...
- {:else if errorFetchApps} -
Error fetching apps
- {:else} - - {/if} -
- -
- Workspace - -
- {/if} - -
- Label (optional) - -
- - {#if !mcpCreationMode} -
- Expires In (optional) - -
- {/if} -
- -
- - -
-
- {/if} -
- -
- - - - prefix - label - expiration - scopes - - - {#snippet body()} - - {#if tokens && tokens.length > 0} - {#each tokens as { token_prefix, expiration, label, scopes }} - - {token_prefix}**** - {label ?? ''} - {displayDate(expiration ?? '')} - {scopes?.join(', ') ?? ''} - - - + +
+ + + + prefix + label + expiration + scopes + + + {#snippet body()} + + {#if tokens && tokens.length > 0} + {#each tokens as { token_prefix, expiration, label, scopes }} + + {token_prefix}**** + {label ?? ''} + {displayDate(expiration ?? '')} + {scopes?.join(', ') ?? ''} + + + + + {/each} + {:else if tokens && tokens.length === 0} + + There are no tokens yet - {/each} - {:else if tokens && tokens.length === 0} - - There are no tokens yet - - {:else} - Loading... - {/if} - - {/snippet} - -
- {#if tokens?.length == 100} - - {/if} - {#if tokenPage > 1} - - {/if} + {:else} + Loading... + {/if} + + {/snippet} + +
+ {#if tokens?.length == 100} + + {/if} + {#if tokenPage > 1} + + {/if} +
diff --git a/frontend/src/lib/components/triggers/DeleteTriggerButton.svelte b/frontend/src/lib/components/triggers/DeleteTriggerButton.svelte index d65c26a01d..b06392bb7e 100644 --- a/frontend/src/lib/components/triggers/DeleteTriggerButton.svelte +++ b/frontend/src/lib/components/triggers/DeleteTriggerButton.svelte @@ -18,7 +18,7 @@ { diff --git a/frontend/src/lib/components/triggers/TriggerEditorToolbar.svelte b/frontend/src/lib/components/triggers/TriggerEditorToolbar.svelte index b54c80e931..ac3032afc1 100644 --- a/frontend/src/lib/components/triggers/TriggerEditorToolbar.svelte +++ b/frontend/src/lib/components/triggers/TriggerEditorToolbar.svelte @@ -85,7 +85,7 @@ />
{/if} - {#if trigger?.isDraft} + {#if trigger?.isDraft || permissions === 'create'} {:else if !trigger?.isDraft && trigger?.draftConfig}
diff --git a/frontend/src/lib/svelte5Utils.svelte.ts b/frontend/src/lib/svelte5Utils.svelte.ts index d45dcbfea2..d738dbf89d 100644 --- a/frontend/src/lib/svelte5Utils.svelte.ts +++ b/frontend/src/lib/svelte5Utils.svelte.ts @@ -1,5 +1,6 @@ // https://github.com/sveltejs/svelte/issues/14600 +import { untrack } from 'svelte' import type { StateStore } from './utils' export function withProps(component: Component, props: Props) { @@ -10,6 +11,11 @@ export function withProps(component: Component, props: Props) return ret } +export function createState(initialValue: T): T { + let s = $state(initialValue) + return s +} + export function stateSnapshot(state: T) { return $state.snapshot(state) } @@ -38,23 +44,25 @@ export function usePromise( status: 'loading', __promise: undefined, refresh: () => { - let promise = createPromise() - ret.__promise = promise - ret.status = 'loading' - if (clearValueOnRefresh) ret.value = undefined - ret.error = undefined + untrack(() => { + let promise = createPromise() + ret.__promise = promise + ret.status = 'loading' + if (clearValueOnRefresh) ret.value = undefined + ret.error = undefined - promise - .then((value) => { - if (ret.__promise !== promise) return - ret.value = value - ret.status = 'ok' - }) - .catch((error) => { - if (ret.__promise !== promise) return - ret.error = error - ret.status = 'error' - }) + promise + .then((value) => { + if (ret.__promise !== promise) return + ret.value = value + ret.status = 'ok' + }) + .catch((error) => { + if (ret.__promise !== promise) return + ret.error = error + ret.status = 'error' + }) + }) } }) if (loadInit) ret.refresh() diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index db01243249..12c4c3b24d 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -1465,3 +1465,20 @@ export function isS3Uri(uri: string): uri is S3Uri { const match = uri.match(/^s3:\/\/([^/]*)\/(.*)$/) return !!match && match.length === 3 } + +export function uniqueBy(array: T[], key: (t: T) => any): T[] { + const seen = new Set() + return array.filter((item) => { + const value = key(item) + if (seen.has(value)) { + return false + } else { + seen.add(value) + return true + } + }) +} + +export function pruneNullishArrayWithSet(array: (T | null | undefined)[]): T[] { + return array.filter((item): item is T => item !== null && item !== undefined) +} diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index df33996942..93e18725d7 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -60,6 +60,9 @@ import { setContext } from 'svelte' import TriggersBadge from '$lib/components/graph/renderers/triggers/TriggersBadge.svelte' import { Triggers } from '$lib/components/triggers/triggers.svelte' + import FlowAssetsHandler, { + initFlowGraphAssetsCtx + } from '$lib/components/flows/FlowAssetsHandler.svelte' let flow: Flow | undefined = $state() let can_write = false @@ -91,6 +94,11 @@ triggersState }) + setContext( + 'FlowGraphAssetContext', + initFlowGraphAssetsCtx({ getModules: () => flow?.value.modules ?? [] }) + ) + let previousPath: string | undefined = $state(undefined) async function archiveFlow(): Promise { @@ -648,3 +656,9 @@ {/snippet} {/if} + + diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index f59f9d8446..1668c7eb0c 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -90,8 +90,12 @@ import CustomPopover from '$lib/components/CustomPopover.svelte' import { isWindmillTooBigObject } from '$lib/components/job_args' import ScheduleEditor from '$lib/components/triggers/schedules/ScheduleEditor.svelte' - import { untrack } from 'svelte' + import { setContext, untrack } from 'svelte' import WorkerHostname from '$lib/components/WorkerHostname.svelte' + import FlowAssetsHandler, { + initFlowGraphAssetsCtx + } from '$lib/components/flows/FlowAssetsHandler.svelte' + import JobAssetsViewer from '$lib/components/assets/JobAssetsViewer.svelte' let job: Job | undefined = $state() let jobUpdateLastFetch: Date | undefined = $state() @@ -99,7 +103,7 @@ let scriptProgress: number | undefined = $state(undefined) let currentJobIsLongRunning: boolean = $state(false) - let viewTab: 'result' | 'logs' | 'code' | 'stats' = $state('result') + let viewTab: 'result' | 'logs' | 'code' | 'stats' | 'assets' = $state('result') let selectedJobStep: string | undefined = $state(undefined) let branchOrIterationN: number = $state(0) @@ -118,6 +122,12 @@ let lastJobId: string | undefined = $state(undefined) let concurrencyKey: string | undefined = $state(undefined) + + setContext( + 'FlowGraphAssetContext', + initFlowGraphAssetsCtx({ getModules: () => job?.raw_flow?.modules ?? [] }) + ) + async function getConcurrencyKey(job: Job | undefined) { if (!job) return lastJobId = job.id @@ -922,6 +932,7 @@ Result Logs Metrics + Assets {#if isScriptPreview(job?.job_kind)} Code {/if} @@ -941,6 +952,10 @@ tag={job?.tag} />
+ {:else if viewTab == 'assets'} +
+ +
{:else if viewTab == 'code'} {#if job && 'raw_code' in job && job.raw_code}
@@ -998,3 +1013,9 @@ {/if}
{/if} + + diff --git a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte index 11b1b3eeb9..f3a0849a7a 100644 --- a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte +++ b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte @@ -64,7 +64,10 @@ } token = undefined - scopes = [`run:${itemKind}/${scriptPath}`] + + + + scopes = [`jobs:run:${itemKind}s:${scriptPath}`] } diff --git a/lsp/Pipfile b/lsp/Pipfile index fc372bb3a5..0499198817 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.507.1" -wmill_pg = ">=1.507.1" +wmill = ">=1.508.0" +wmill_pg = ">=1.508.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 3eb17a5326..0de884f00a 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.507.1 + version: 1.508.0 title: OpenFlow Spec contact: name: Ruben Fiszel @@ -270,7 +270,7 @@ components: type: string is_trigger: type: boolean - asset_fallback_access_types: + assets: type: array items: type: object @@ -287,10 +287,10 @@ components: - resource access_type: type: string - enum: - - r - - w - - rw + enum: [r, w, rw] + alt_access_type: + type: string + enum: [r, w, rw] required: - type - content diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index c5033eb071..08c5fa5274 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.507.1' + ModuleVersion = '1.508.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index a5760b2f56..d73660430e 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.507.1" +version = "1.508.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 987e5ec3a9..8ce27e3a66 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.507.1" +version = "1.508.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 6f071b168b..93e0c9dd23 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.507.1", + "version": "1.508.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 1417a4f461..92b45520db 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.507.1", + "version": "1.508.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 0f5f07a749..28bcf0e708 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.507.1 +1.508.0