starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-19 08:03:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

run

Browse Source
This commit is contained in:
Ruben Fiszel
2025-07-17 23:26:51 +00:00
parent 7c7b9b3768 2ab5345e61
commit 94eb6e14f4
170 changed files with 5023 additions and 2010 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/CODEOWNERS
+3 -3
View File
@@ -1,4 +1,4 @@
* @rubenfiszel @HugoCasa @alpetric
* @rubenfiszel @hugocasa @alpetric
/community/ @rubenfiszel @HugoCasa @alpetric
/frontend/ @rubenfiszel @HugoCasa @alpetric
/community/ @rubenfiszel @hugocasa @alpetric
/frontend/ @rubenfiszel @hugocasa @alpetric
.github/DockerfileBackendTests
+1 -1
View File
@@ -42,7 +42,7 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER
RUN /usr/local/bin/python3 -m pip install pip-tools
# Bun
COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun
ARG TARGETPLATFORM
.github/workflows/build_windows_worker_.yml
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
- name: Setup Rust
uses: actions-rs/toolchain@v1
with:
toolchain: 1.85.0
toolchain: 1.88.0
override: true
- name: Substitute EE code
.github/workflows/git-commands.yaml
+5 -5
View File
@@ -151,17 +151,17 @@ jobs:
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
echo "${{ steps.get-commit-hash.outputs.commit_hash }}" > backend/ee-repo-ref.txt
echo "Updated backend/ee-repo-ref.txt with commit hash: ${{ steps.get-commit-hash.outputs.commit_hash }}"
# commit and push the changes
PR_NUMBER=${{ github.event.issue.number }}
BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)
echo "Checking out PR branch: $BRANCH_NAME"
git checkout $BRANCH_NAME
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
git config pull.rebase true
git pull origin $BRANCH_NAME
echo "${{ steps.get-commit-hash.outputs.commit_hash }}" > backend/ee-repo-ref.txt
echo "Updated backend/ee-repo-ref.txt with commit hash: ${{ steps.get-commit-hash.outputs.commit_hash }}"
# commit and push the changes
PR_NUMBER=${{ github.event.issue.number }}
BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)
git add backend/ee-repo-ref.txt
git commit -m "Update ee-repo-ref.txt" || echo "No changes to commit"
git push origin $BRANCH_NAME
.github/workflows/publish_windows_worker.yml
+1 -1
View File
@@ -35,7 +35,7 @@ jobs:
- name: Setup Rust
uses: actions-rs/toolchain@v1
with:
toolchain: 1.85.0
toolchain: 1.88.0
override: true
- name: Substitute EE code
CHANGELOG.md
+24
View File
@@ -1,5 +1,29 @@
# Changelog
## [1.508.0](https://github.com/windmill-labs/windmill/compare/v1.507.2...v1.508.0) (2025-07-16)
### Features
* **aichat:** add logs api endpoint as tool ([#6197](https://github.com/windmill-labs/windmill/issues/6197)) ([827e06b](https://github.com/windmill-labs/windmill/commit/827e06b4b3dd7b2b970b3a0a37e3dfbe98d1fafd))
* better explanation for dev key renewal ([#6209](https://github.com/windmill-labs/windmill/issues/6209)) ([1d62dd0](https://github.com/windmill-labs/windmill/commit/1d62dd0fc323e484ab1d08e740379c8243b52df8))
### Bug Fixes
* **frontend:** add error handler on trigger delete ([#6208](https://github.com/windmill-labs/windmill/issues/6208)) ([c076d13](https://github.com/windmill-labs/windmill/commit/c076d1332e0871b16b5881e2cf3ef165bc9f5dd5))
* **frontend:** enable delete triggers on detail page ([#6206](https://github.com/windmill-labs/windmill/issues/6206)) ([290daec](https://github.com/windmill-labs/windmill/commit/290daec0fa53f77f6ad02e746d82aa30997d8cb0))
* hide s3 catalog picker in anonymous apps ([#6204](https://github.com/windmill-labs/windmill/issues/6204)) ([b68193f](https://github.com/windmill-labs/windmill/commit/b68193f80468aeeadff5af0826cca2f0cc9f05e8))
* support enum for array list as multiselect ([07b785d](https://github.com/windmill-labs/windmill/commit/07b785dcc383ae87eaec65e765177e45fff0af20))
## [1.507.2](https://github.com/windmill-labs/windmill/compare/v1.507.1...v1.507.2) (2025-07-16)
### Bug Fixes
* cancel autocomplete on escape + autocomplete qol ([#6201](https://github.com/windmill-labs/windmill/issues/6201)) ([038c179](https://github.com/windmill-labs/windmill/commit/038c179a432d4f1997a761392e27b16dea829e06))
* **cli:** fix cli --skip-resources --skip-variables ([6c477c1](https://github.com/windmill-labs/windmill/commit/6c477c109bf7e803ec07d4485dec426b770c7f76))
## [1.507.1](https://github.com/windmill-labs/windmill/compare/v1.507.0...v1.507.1) (2025-07-16)
Dockerfile
+1 -1
View File
@@ -194,7 +194,7 @@ COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.2.18 /usr/local/bin/bun /usr/bin/bun
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
backend/.sqlx/query-00ec0814dc270ee08d8cd818815681862711af1e6b046c4ff479622c153b69ed.json
Generated
+68
View File
@@ -0,0 +1,68 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE\n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n CASE\n -- flow step:\n WHEN flow_step_id IS NOT NULL THEN NULL\n -- completed:\n WHEN c.id IS NOT NULL THEN COALESCE(\n c.workflow_as_code_status || c.flow_status,\n c.workflow_as_code_status,\n c.flow_status\n )\n -- not completed:\n ELSE COALESCE(\n f.workflow_as_code_status || f.flow_status,\n f.workflow_as_code_status,\n f.flow_status\n )\n END AS \"flow_status: sqlx::types::Json<Box<RawValue>>\",\n job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "completed",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "running",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "logs",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "mem_peak",
"type_info": "Int4"
},
{
"ordinal": 4,
"name": "flow_status: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "log_offset",
"type_info": "Int4"
},
{
"ordinal": 6,
"name": "created_by!",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "progress",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Int4",
"Text",
"Uuid",
"Bool",
"Bool"
]
},
"nullable": [
null,
null,
null,
null,
null,
null,
false,
null
]
},
"hash": "00ec0814dc270ee08d8cd818815681862711af1e6b046c4ff479622c153b69ed"
}
backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861.json → backend/.sqlx/query-140f4ad799fca6c25975a0aca7c9051f0760e22eccdd291c83ed86599ce571cb.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_status (id, workflow_as_code_status)\n VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3))\n ON CONFLICT (id) DO UPDATE SET\n workflow_as_code_status = JSONB_SET(\n COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB), \n array[$2],\n $3\n )",
"query": "INSERT INTO v2_job_status (id, workflow_as_code_status)\n VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3))\n ON CONFLICT (id) DO UPDATE SET\n workflow_as_code_status = JSONB_SET(\n COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB),\n array[$2],\n $3\n )",
"describe": {
"columns": [],
"parameters": {
@@ -12,5 +12,5 @@
},
"nullable": []
},
"hash": "2e6935811a6d818bc523f076674f794f8be6c6bad3d06e74586e8ab668d91861"
"hash": "140f4ad799fca6c25975a0aca7c9051f0760e22eccdd291c83ed86599ce571cb"
}
backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json
Generated
+91
View File
@@ -0,0 +1,91 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Int8",
"Varchar",
"Int8Array",
"Text",
"Text",
"Text",
"Varchar",
"Text",
"Bool",
"Jsonb",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb"
]
}
}
},
{
"Custom": {
"name": "script_kind",
"kind": {
"Enum": [
"script",
"trigger",
"failure",
"command",
"approval",
"preprocessor"
]
}
}
},
"Varchar",
"Bool",
"VarcharArray",
"Int4",
"Int4",
"Int4",
"Bool",
"Bool",
"Int2",
"Bool",
"Bool",
"Int4",
"Varchar",
"Bool",
"Bool",
"Varchar",
"Bool",
"Text",
"Bool",
"Jsonb"
]
},
"nullable": []
},
"hash": "1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e"
}
backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33.json → backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json<Box<RawValue>>\"\n FROM v2_as_completed_job \n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))",
"query": "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json<Box<RawValue>>\"\n FROM v2_as_completed_job\n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))",
"describe": {
"columns": [
{
@@ -26,5 +26,5 @@
true
]
},
"hash": "47fb4bceddd36fa60ccbe84f6341436fb133f0b3c97abb1267a9c12326dd1a33"
"hash": "2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9"
}
backend/.sqlx/query-95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a.json → backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_as_queue \n LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id \n WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))",
"query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_as_queue\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id\n WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))",
"describe": {
"columns": [
{
@@ -38,5 +38,5 @@
true
]
},
"hash": "95ae90094ec0e2c22660cc2e3788b22231dab9c558723cc54894597ce4cd3d5a"
"hash": "2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f"
}
backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-05cb171b610bfb45f6228128a385cde8a5b86d7ca377a028004cc382e12faf41.json → backend/.sqlx/query-7b5dc50b685afc7de0b99bd68e2217d660529c15b41860a4513f63d465bb9abb.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO concurrency_counter(concurrency_id, job_uuids) \n VALUES ($1, '{}'::jsonb)",
"query": "INSERT INTO concurrency_counter(concurrency_id, job_uuids)\n VALUES ($1, '{}'::jsonb)",
"describe": {
"columns": [],
"parameters": {
@@ -10,5 +10,5 @@
},
"nullable": []
},
"hash": "05cb171b610bfb45f6228128a385cde8a5b86d7ca377a028004cc382e12faf41"
"hash": "7b5dc50b685afc7de0b99bd68e2217d660529c15b41860a4513f63d465bb9abb"
}
backend/.sqlx/query-625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31.json → backend/.sqlx/query-81ba1e003d3f6e8e7c74a4dd09b980888d71121082390f3fdb91269cd0c5d0b2.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json<Box<RawValue>>\" FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n LEFT JOIN flow_version_lite \n ON flow_version_lite.id = flow_version.id\n WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1",
"query": "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json<Box<RawValue>>\" FROM flow\n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n LEFT JOIN flow_version_lite\n ON flow_version_lite.id = flow_version.id\n WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1",
"describe": {
"columns": [
{
@@ -19,5 +19,5 @@
null
]
},
"hash": "625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31"
"hash": "81ba1e003d3f6e8e7c74a4dd09b980888d71121082390f3fdb91269cd0c5d0b2"
}
backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378.json
Generated
-68
View File
@@ -1,68 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE \n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n CASE\n -- flow step:\n WHEN flow_step_id IS NOT NULL THEN NULL\n -- completed:\n WHEN c.id IS NOT NULL THEN COALESCE(\n c.workflow_as_code_status || c.flow_status,\n c.workflow_as_code_status,\n c.flow_status\n )\n -- not completed:\n ELSE COALESCE(\n f.workflow_as_code_status || f.flow_status,\n f.workflow_as_code_status,\n f.flow_status\n )\n END AS \"flow_status: sqlx::types::Json<Box<RawValue>>\",\n job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "completed",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "running",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "logs",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "mem_peak",
"type_info": "Int4"
},
{
"ordinal": 4,
"name": "flow_status: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 5,
"name": "log_offset",
"type_info": "Int4"
},
{
"ordinal": 6,
"name": "created_by!",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "progress",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Int4",
"Text",
"Uuid",
"Bool",
"Bool"
]
},
"nullable": [
null,
null,
null,
null,
null,
null,
false,
null
]
},
"hash": "b075c77caad37cdd75faab1d934b7a63521c1ad788033c564ffd7a2944454378"
}
backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json → backend/.sqlx/query-b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT \n j.id,\n j.kind AS \"kind: _\",\n COALESCE(s.path, f.path) AS \"script_path!\",\n COALESCE(s.hash, f.id) AS \"script_hash!: _\",\n COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS \"scheduled_for!: _\",\n args AS input,\n COALESCE(s.schema, f.schema) AS \"schema: _\"\n FROM v2_job j\n LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n LEFT JOIN v2_job_completed jc ON jc.id = j.id\n LEFT JOIN v2_job_queue jq ON jq.id = j.id\n WHERE j.id = ANY($1)\n AND j.workspace_id = $2\n AND COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL",
"query": "SELECT\n j.id,\n j.kind AS \"kind: _\",\n COALESCE(s.path, f.path) AS \"script_path!\",\n COALESCE(s.hash, f.id) AS \"script_hash!: _\",\n COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS \"scheduled_for!: _\",\n args AS input,\n COALESCE(s.schema, f.schema) AS \"schema: _\"\n FROM v2_job j\n LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n LEFT JOIN v2_job_completed jc ON jc.id = j.id\n LEFT JOIN v2_job_queue jq ON jq.id = j.id\n WHERE j.id = ANY($1)\n AND j.workspace_id = $2\n AND COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL",
"describe": {
"columns": [
{
@@ -81,5 +81,5 @@
null
]
},
"hash": "ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9"
"hash": "b3c02fd225a6aa78785d466e7f033b38deb9c7fa17bd3836c9ad8884f27be84a"
}
backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c.json → backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job \n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id \n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))",
"query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))",
"describe": {
"columns": [
{
@@ -38,5 +38,5 @@
true
]
},
"hash": "2c5a1b1ebd872ce8a03a1b7e5246c876750c85075cfda816c025c805d4c3cd4c"
"hash": "ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9"
}
backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json → backend/.sqlx/query-cad2dfc50d57c2298cb0ec05ed3b42b9fd3edd05c314c9d96ba33cad37fe35c2.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8",
"query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id)\n SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8",
"describe": {
"columns": [],
"parameters": {
@@ -17,5 +17,5 @@
},
"nullable": []
},
"hash": "fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa"
"hash": "cad2dfc50d57c2298cb0ec05ed3b42b9fd3edd05c314c9d96ba33cad37fe35c2"
}
backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836.json → backend/.sqlx/query-f6e63a324664423353b56bccd9f3b322a3512ccfa86485e0f01664942ae04d0f.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "WITH uuid_table as (\n select unnest($4::uuid[]) as uuid\n )\n INSERT INTO v2_job_queue\n (id, workspace_id, scheduled_for, tag)\n (SELECT uuid, $1, $2, $3 FROM uuid_table) \n RETURNING id",
"query": "WITH uuid_table as (\n select unnest($4::uuid[]) as uuid\n )\n INSERT INTO v2_job_queue\n (id, workspace_id, scheduled_for, tag)\n (SELECT uuid, $1, $2, $3 FROM uuid_table)\n RETURNING id",
"describe": {
"columns": [
{
@@ -21,5 +21,5 @@
false
]
},
"hash": "70e740465e648d84d32a506916ed437d1e32d4b5bdc7f7fae22e804428660836"
"hash": "f6e63a324664423353b56bccd9f3b322a3512ccfa86485e0f01664942ae04d0f"
}
backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/Cargo.lock
Generated
+38 -37
View File
@@ -776,9 +776,9 @@ dependencies = [
[[package]]
name = "aws-credential-types"
version = "1.2.3"
version = "1.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "687bc16bc431a8533fe0097c7f0182874767f920989d7260950172ae8e3c4465"
checksum = "b68c2194a190e1efc999612792e25b1ab3abfefe4306494efaaabc25933c0cbe"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
@@ -835,9 +835,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sqs"
version = "1.75.0"
version = "1.76.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c27b0a5c8f9f8db35929bde0612d738d8c872ab887232579c61d73dba7a5d17"
checksum = "837b4b181a1306a5a67d85218fa7d7298e340041d9bd81ae6919b9ec4f86ca97"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -857,9 +857,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
version = "1.75.0"
version = "1.76.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36d2a4d2753a2304cc0a86cff15581179c420c7b3054c0cbc884a2e497e79ae5"
checksum = "64bf26698dd6d238ef1486bdda46f22a589dc813368ba868dc3d94c8d27b56ba"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -879,9 +879,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
version = "1.76.0"
version = "1.77.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719f007d6914388e2f14943c663447cfc44bb9d0478bb96df891258ba5145f82"
checksum = "09cd07ed1edd939fae854a22054299ae3576500f4e0fadc560ca44f9c6ea1664"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -901,9 +901,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
version = "1.77.0"
version = "1.78.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2efcc3dbb630f3e0d16a4f79fcd4b3fb1c59cb89f1031c46e44976f640b5de27"
checksum = "37f7766d2344f56d10d12f3c32993da36d78217f32594fe4fb8e57a538c1cdea"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -14676,7 +14676,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"axum",
@@ -14728,7 +14728,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"argon2",
@@ -14839,7 +14839,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"base64 0.22.1",
"chrono",
@@ -14854,7 +14854,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"chrono",
"serde",
@@ -14867,7 +14867,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"serde",
@@ -14881,7 +14881,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"async-stream",
@@ -14934,6 +14934,7 @@ dependencies = [
"sqlx",
"strum 0.27.1",
"strum_macros 0.27.1",
"sysinfo",
"systemstat",
"tar",
"tempfile",
@@ -14959,7 +14960,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"regex",
"serde",
@@ -14973,7 +14974,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"bytes",
@@ -14996,7 +14997,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -15008,7 +15009,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -15017,7 +15018,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15029,7 +15030,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15041,7 +15042,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"gosyn",
@@ -15053,7 +15054,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15065,7 +15066,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15077,7 +15078,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -15088,7 +15089,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15099,7 +15100,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15111,7 +15112,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15134,7 +15135,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -15151,7 +15152,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15165,7 +15166,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15183,7 +15184,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"getrandom 0.2.16",
@@ -15207,7 +15208,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15217,7 +15218,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15250,7 +15251,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"wasm-bindgen",
"wasm-bindgen-test",
@@ -15260,7 +15261,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.507.1"
version = "1.508.0"
dependencies = [
"anyhow",
"async-recursion",
backend/Cargo.toml
+3 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.507.1"
version = "1.508.0"
authors.workspace = true
edition.workspace = true
@@ -32,7 +32,7 @@ members = [
]
[workspace.package]
version = "1.507.1"
version = "1.508.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -267,6 +267,7 @@ google-cloud-googleapis = {version = "0.16.1", features = ["pubsub"]}
# TODO: remove once deno fixes the issue on their end
# https://github.com/denoland/deno/issues/28557
winapi = { version = "0.3.9", features = ["sysinfoapi"] }
sysinfo = { version = "0.32.1" }
swc_common = "=0.37.5"
swc_ecma_parser = "=0.149.1"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
3cccbe81494f7edd113dce2812278b00c91160c0
773a959358d0dc30255621a584fc2b8e2dd41582
backend/migrations/20250717135503_assets_refactor.down.sql
+7
View File
@@ -0,0 +1,7 @@
-- Add 'variable' kind back
ALTER TABLE asset ALTER column kind TYPE VARCHAR;
DROP TYPE asset_kind;
CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource', 'variable');
ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND;
ALTER TABLE script DROP COLUMN assets;
backend/migrations/20250717135503_assets_refactor.up.sql
+8
View File
@@ -0,0 +1,8 @@
ALTER TABLE script ADD COLUMN assets jsonb;
-- Remove 'variable' kind
DELETE FROM asset WHERE kind = 'variable';
ALTER TABLE asset ALTER column kind TYPE VARCHAR;
DROP TYPE asset_kind;
CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource');
ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND;
backend/parsers/windmill-parser-py/src/asset_parser.rs
+18 -18
View File
@@ -57,27 +57,27 @@ impl AssetsFinder {
})
.ok_or(())?;
use AssetKind::*;
let (kind, access_type, arg) = match ident.as_str() {
"load_s3_file" => (AssetKind::S3Object, Some(R), Arg::Pos(0)),
"load_s3_file_reader" => (AssetKind::S3Object, Some(R), Arg::Pos(0)),
"write_s3_file" => (AssetKind::S3Object, Some(W), Arg::Pos(0)),
"get_resource" => (AssetKind::Resource, None, Arg::Pos(0)),
"set_resource" => (AssetKind::Resource, Some(W), Arg::Named("path")),
"get_boto3_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)),
"get_polars_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)),
"get_duckdb_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)),
"get_variable" => (AssetKind::Variable, Some(R), Arg::Pos(0)),
"set_variable" => (AssetKind::Variable, Some(W), Arg::Pos(0)),
"load_s3_file" => (S3Object, Some(R), Arg::Pos(0, "s3object")),
"load_s3_file_reader" => (S3Object, Some(R), Arg::Pos(0, "s3object")),
"write_s3_file" => (S3Object, Some(W), Arg::Pos(0, "s3object")),
"get_resource" => (Resource, None, Arg::Pos(0, "path")),
"set_resource" => (Resource, Some(W), Arg::Pos(0, "path")),
"get_boto3_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")),
"get_polars_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")),
"get_duckdb_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")),
_ => return Err(()),
};
let arg_val = match arg {
Arg::Pos(i) => node.args.get(i),
Arg::Named(name) => node
.keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some(name))
.map(|kw| &kw.value),
Arg::Pos(i, name) => node.args.get(i).or_else(|| {
// Get arg by name
node.keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some(name))
.map(|kw| &kw.value)
}),
};
match arg_val {
@@ -93,6 +93,6 @@ impl AssetsFinder {
}
enum Arg {
Pos(usize),
Named(&'static str),
// Positional arguments in python can also be used by their name
Pos(usize, &'static str),
}
backend/parsers/windmill-parser-ts/src/asset_parser.rs
-2
View File
@@ -84,8 +84,6 @@ impl AssetsFinder {
"denoS3LightClientSettings" => (AssetKind::Resource, None, 0),
"duckdbConnectionSettings" => (AssetKind::Resource, None, 0),
"polarsConnectionSettings" => (AssetKind::Resource, None, 0),
"getVariable" => (AssetKind::Variable, Some(R), 0),
"setVariable" => (AssetKind::Variable, Some(W), 0),
_ => return Err(()),
};
backend/parsers/windmill-parser/src/asset_parser.rs
-3
View File
@@ -15,7 +15,6 @@ use AssetUsageAccessType::*;
pub enum AssetKind {
S3Object,
Resource,
Variable,
}
#[derive(Serialize)]
@@ -57,8 +56,6 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> {
Some((AssetKind::Resource, &s[6..]))
} else if s.starts_with("$res:") {
Some((AssetKind::Resource, &s[5..]))
} else if s.starts_with("var://") {
Some((AssetKind::Variable, &s[6..]))
} else {
None
}
backend/substitute_ee_code.sh
+1 -1
View File
@@ -99,4 +99,4 @@ if [ "$MOVE_NEW_FILES" == "YES" ]; then
echo "File moved '${ce_file}' -->> '${ee_file}'"
fi
done
fi
fi
backend/tests/worker.rs
+6 -6
View File
@@ -1147,7 +1147,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}
.into(),
stop_after_if: Default::default(),
@@ -1191,7 +1191,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}
.into(),
stop_after_if: Default::default(),
@@ -1321,7 +1321,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}.into(),
stop_after_if: Default::default(),
@@ -1376,7 +1376,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
@@ -1416,7 +1416,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}.into(),
stop_after_if: Default::default(),
@@ -1482,7 +1482,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
backend/update_sqlx.sh
+2 -2
View File
@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Default directory
EE_DIR="../windmill-ee-private"
@@ -32,4 +32,4 @@ if [[ "$(uname)" == "Darwin" ]]; then
sed -i '' 's/^#samael = { version="0.0.14", features = \["xmlsec"\] }/samael = { version="0.0.14", features = ["xmlsec"] }/' Cargo.toml
# Comment out the git-based samael dependency
sed -i '' 's/^\(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/# \1/' Cargo.toml
fi
fi
backend/windmill-api/openapi.yaml
+61 -5
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.507.1
version: 1.508.0
title: Windmill API
contact:
@@ -2792,6 +2792,22 @@ paths:
schema:
type: string
/tokens/list/scopes:
get:
summary: list of available scopes
operationId: ListAvailableScopes
tags:
- token
responses:
"200":
description: list of available scopes
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/ScopeDomain"
/users/tokens/create:
post:
summary: create token
@@ -13856,20 +13872,27 @@ components:
type: boolean
on_behalf_of_email:
type: string
fallback_access_types:
assets:
type: array
items:
type: object
required: [path, kind, access_type]
required:
- path
- kind
properties:
path:
type: string
kind:
type: string
enum: [s3object, resource]
enum:
- s3object
- resource
access_type:
type: string
enum: [r, w, rw]
alt_access_type:
type: string
enum: [r, w, rw]
required:
- path
@@ -16771,6 +16794,40 @@ components:
- edited_at
- execution_mode
ScopeDefinition:
type: object
properties:
value:
type: string
label:
type: string
description:
type: string
nullable: true
requires_resource_path:
type: boolean
required:
- value
- label
- requires_resource_path
ScopeDomain:
type: object
properties:
name:
type: string
description:
type: string
nullable: true
scopes:
type: array
items:
$ref: "#/components/schemas/ScopeDefinition"
required:
- name
- scopes
ListableRawApp:
type: object
properties:
@@ -17590,7 +17647,6 @@ components:
enum:
- s3object
- resource
- variable
Asset:
type: object
properties:
backend/windmill-api/src/apps.rs
+27 -8
View File
@@ -13,7 +13,7 @@ use crate::{
db::{ApiAuthed, DB},
resources::get_resource_value_interpolated_internal,
users::{require_owner_of_path, OptAuthed},
utils::WithStarredInfoQuery,
utils::{check_scopes, WithStarredInfoQuery},
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
};
@@ -444,6 +444,7 @@ async fn get_app(
Query(query): Query<WithStarredInfoQuery>,
) -> JsonResult<AppWithLastVersionAndStarred> {
let path = path.to_path();
check_scopes(&authed, || format!("apps:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let app_o = if query.with_starred_info.unwrap_or(false) {
@@ -491,6 +492,7 @@ async fn get_app_lite(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<AppWithLastVersion> {
let path = path.to_path();
check_scopes(&authed, || format!("apps:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
@@ -518,6 +520,7 @@ async fn get_app_w_draft(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<AppWithLastVersionAndDraft> {
let path = path.to_path();
check_scopes(&authed, || format!("apps:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let app_o = sqlx::query_as::<_, AppWithLastVersionAndDraft>(
@@ -547,6 +550,8 @@ async fn get_app_history(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Vec<AppHistory>> {
let path = path.to_path();
check_scopes(&authed, || format!("apps:read:{}", &path))?;
let mut tx = user_db.begin(&authed).await?;
let query_result = sqlx::query!(
"SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg
@@ -554,7 +559,7 @@ async fn get_app_history(
WHERE a.workspace_id = $1 AND a.path = $2
ORDER BY created_at DESC",
w_id,
path.to_path(),
path,
).fetch_all(&mut *tx).await?;
tx.commit().await?;
@@ -574,6 +579,8 @@ async fn get_latest_version(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Option<AppHistory>> {
let path = path.to_path();
check_scopes(&authed, || format!("apps:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let row = sqlx::query!(
"SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg
@@ -581,7 +588,7 @@ async fn get_latest_version(
WHERE a.workspace_id = $1 AND a.path = $2
ORDER BY created_at DESC",
w_id,
path.to_path(),
path,
).fetch_optional(&mut *tx).await?;
tx.commit().await?;
@@ -609,17 +616,19 @@ async fn update_app_history(
.fetch_optional(&mut *tx)
.await?;
if app_path.is_none() {
let Some(app_path) = app_path else {
tx.commit().await?;
return Err(Error::NotFound(
format!("App with ID {app_id} not found").to_string(),
));
}
};
check_scopes(&authed, || format!("apps:write:{}", &app_path))?;
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, app_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET deployment_msg = $4",
w_id,
app_path.unwrap(),
app_path,
app_version,
app_history_update.deployment_msg,
)
@@ -664,6 +673,9 @@ async fn get_app_by_id(
tx.commit().await?;
let app = not_found_if_none(app_o, "App", id.to_string())?;
check_scopes(&authed, || format!("apps:read:{}", &app.path))?;
Ok(Json(app))
}
@@ -756,6 +768,7 @@ async fn get_secret_id(
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("apps:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let id_o = sqlx::query_scalar!(
@@ -861,6 +874,8 @@ async fn create_app_raw<'a>(
)
.await?;
check_scopes(&authed, || format!("apps:write:{}", path))?;
webhook.send_message(
w_id.clone(),
WebhookMessage::CreateApp { workspace: w_id, path: path.clone() },
@@ -899,6 +914,8 @@ async fn create_app(
Json(app): Json<CreateApp>,
) -> Result<(StatusCode, String)> {
let path = app.path.clone();
check_scopes(&authed, || format!("apps:write:{}", &path))?;
let (new_tx, _path, _id) = create_app_internal(authed, db, user_db, &w_id, false, app).await?;
new_tx.commit().await?;
@@ -1101,6 +1118,7 @@ async fn delete_app(
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("apps:write:{}", path))?;
if path == "g/all/setup_app" && w_id == "admins" {
return Err(Error::BadRequest(
@@ -1184,6 +1202,7 @@ async fn update_app(
) -> Result<String> {
// create_app_internal(authed, user_db, db, &w_id, &mut app).await?;
let path = path.to_path();
check_scopes(&authed, || format!("apps:write:{}", path))?;
let opath = path.to_string();
let (new_tx, npath, _v_id) =
update_app_internal(authed, db, user_db, &w_id, path, false, ns).await?;
@@ -1210,6 +1229,7 @@ async fn update_app_raw<'a>(
multipart: Multipart,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("apps:write:{}", path))?;
let opath = path.to_string();
let (npath, _id) = process_app_multipart!(
authed,
@@ -2421,10 +2441,9 @@ async fn exists_app(
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<bool> {
let path = path.to_path();
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
path,
path.to_path(),
w_id
)
.fetch_one(&db)
backend/windmill-api/src/auth.rs
+57 -21
View File
@@ -22,6 +22,7 @@ use tokio::sync::RwLock;
use windmill_common::{
auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, TOKEN_PREFIX_LEN},
error::Error,
jwt,
users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL},
};
@@ -218,7 +219,9 @@ impl AuthCache {
folders,
scopes: None,
username_override,
token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()),
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
})
} else {
let groups = vec![name.to_string()];
@@ -240,7 +243,9 @@ impl AuthCache {
folders,
scopes: None,
username_override,
token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()),
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
})
}
} else {
@@ -303,7 +308,9 @@ impl AuthCache {
folders,
scopes,
username_override,
token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()),
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
})
}
None if super_admin => Some(ApiAuthed {
@@ -315,7 +322,9 @@ impl AuthCache {
folders: vec![],
scopes,
username_override,
token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()),
token_prefix: Some(
token[0..TOKEN_PREFIX_LEN].to_string(),
),
}),
None => None,
}
@@ -491,6 +500,33 @@ where
}
}
pub fn transform_old_scope_to_new_scope(scopes: Option<&mut Vec<String>>) {
if let Some(scopes) = scopes {
for scope in scopes.iter_mut() {
if scope.starts_with("run:") {
let (_, part_scope) = scope.split_once(":").unwrap();
if let Some((kind, path)) = part_scope.split_once("/") {
//appending a 's' as runnable kind is singular while new scope format expect it to be plural
*scope = format!("jobs:run:{}s:{}", kind, path);
}
} else if scope.starts_with("jobs:") {
// Map old jobs scopes to new format
let new_scope = match scope.as_str() {
"jobs:listjobs" => "jobs:read",
"jobs:runscript" => "jobs:run:scripts",
"jobs:runflow" => "jobs:run:flows",
"jobs:resumeflow" => "jobs:run:flows",
"jobs:deletejob" => "jobs:write",
_ => continue,
};
*scope = new_scope.to_string();
}
}
}
}
fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option<String> {
let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" {
Some(path_vec[3].to_owned())
@@ -517,7 +553,7 @@ impl<S> FromRequestParts<S> for ApiAuthed
where
S: Send + Sync,
{
type Rejection = (StatusCode, String);
type Rejection = Error;
async fn from_request_parts(
parts: &mut Parts,
@@ -538,7 +574,6 @@ where
} else {
extract_token(parts, state).await
};
if let Some(token) = token_o {
if let Ok(Extension(cache)) =
Extension::<Arc<AuthCache>>::from_request_parts(parts, state).await
@@ -551,21 +586,22 @@ where
let path_vec: Vec<&str> = original_uri.path().split("/").collect();
let workspace_id = maybe_get_workspace_id_from_path(&path_vec);
if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await {
if authed.scopes.as_ref().is_some_and(|scopes| {
scopes
.iter()
.any(|s| s.starts_with("jobs:") || s.starts_with("run:"))
}) && (path_vec.len() < 3
|| (path_vec[4] != "jobs" && path_vec[4] != "jobs_u"))
{
BRUTE_FORCE_COUNTER.increment().await;
return Err((
StatusCode::UNAUTHORIZED,
format!("Unauthorized scoped token: {:?}", authed.scopes),
));
}
if let Some(mut authed) = cache.get_authed(workspace_id.clone(), &token).await {
if authed.scopes.is_some() {
transform_old_scope_to_new_scope(authed.scopes.as_mut());
let path = original_uri.path();
let method = parts.method.as_str();
if let Err(err) = crate::scopes::check_scopes_for_route(
authed.scopes.as_deref(),
path,
method,
) {
BRUTE_FORCE_COUNTER.increment().await;
return Err(err);
}
}
parts.extensions.insert(authed.clone());
Span::current().record("username", &authed.username.as_str());
@@ -579,7 +615,7 @@ where
}
}
BRUTE_FORCE_COUNTER.increment().await;
Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
Err(Error::NotAuthorized("Unauthorized".to_string()))
}
}
backend/windmill-api/src/concurrency_groups.rs
+8 -6
View File
@@ -1,9 +1,11 @@
use crate::db::{ApiAuthed, DB};
use crate::jobs::{
filter_list_completed_query, filter_list_queue_query, Job, ListCompletedQuery, ListQueueQuery,
UnifiedJob,
use crate::{
db::{ApiAuthed, DB},
jobs::{
filter_list_completed_query, filter_list_queue_query, Job, ListCompletedQuery,
ListQueueQuery, UnifiedJob,
},
utils::check_scopes,
};
use crate::users::check_scopes;
use axum::extract::Path;
use axum::routing::{delete, get};
use axum::{extract::Query, Extension, Json};
@@ -144,7 +146,7 @@ async fn get_concurrent_intervals(
Query(iq): Query<ExtendedJobsParams>,
Query(lq): Query<ListCompletedQuery>,
) -> JsonResult<ExtendedJobs> {
check_scopes(&authed, || format!("jobs:listjobs"))?;
check_scopes(&authed, || format!("jobs:read"))?;
if lq.success.is_some() && lq.running.is_some_and(|x| x) {
return Err(error::Error::BadRequest(
backend/windmill-api/src/flows.rs
+21 -9
View File
@@ -8,15 +8,15 @@
use std::collections::HashMap;
use crate::db::ApiAuthed;
use crate::triggers::{
get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail,
};
use crate::utils::WithStarredInfoQuery;
use crate::{
db::DB,
db::{ApiAuthed, DB},
schedule::clear_schedule,
triggers::{
get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail,
},
users::{maybe_refresh_folders, require_owner_of_path},
utils::check_scopes,
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
};
@@ -333,6 +333,8 @@ async fn list_paths_from_workspace_runnable(
Extension(user_db): Extension<UserDB>,
Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>,
) -> JsonResult<Vec<String>> {
let path = path.to_path();
check_scopes(&authed, || format!("flows:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let runnables = sqlx::query_scalar!(
r#"SELECT f.path
@@ -340,7 +342,7 @@ async fn list_paths_from_workspace_runnable(
JOIN flow f
ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id
WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#,
path.to_path(),
path,
matches!(runnable_kind, RunnableKind::Flow),
w_id
)
@@ -358,6 +360,7 @@ async fn create_flow(
Path(w_id): Path<String>,
Json(nf): Json<NewFlow>,
) -> Result<(StatusCode, String)> {
check_scopes(&authed, || format!("flows:write:{}", nf.path))?;
if *CLOUD_HOSTED {
let nb_flows =
sqlx::query_scalar!("SELECT COUNT(*) FROM flow WHERE workspace_id = $1", &w_id)
@@ -579,6 +582,7 @@ async fn get_flow_history(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Vec<FlowVersion>> {
let path = path.to_path();
check_scopes(&authed, || format!("flows:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let flows = sqlx::query_as!(
@@ -603,6 +607,7 @@ async fn get_latest_version(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Option<FlowVersion>> {
let path = path.to_path();
check_scopes(&authed, || format!("flows:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let version = sqlx::query_as!(
@@ -627,6 +632,7 @@ async fn get_flow_version(
Path((w_id, version, path)): Path<(String, i64, StripPath)>,
) -> JsonResult<Flow> {
let path = path.to_path();
check_scopes(&authed, || format!("flows:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let flow = sqlx::query_as::<_, Flow>(
@@ -660,6 +666,7 @@ async fn update_flow_history(
Json(history_update): Json<FlowHistoryUpdate>,
) -> Result<()> {
let path = path.to_path();
check_scopes(&authed, || format!("flows:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let path_o = sqlx::query_scalar!(
"SELECT flow.path FROM flow
@@ -701,6 +708,8 @@ async fn update_flow(
Path((w_id, flow_path)): Path<(String, StripPath)>,
Json(nf): Json<NewFlow>,
) -> Result<String> {
let flow_path = flow_path.to_path();
check_scopes(&authed, || format!("flows:write:{}", flow_path))?;
#[cfg(not(feature = "enterprise"))]
if nf
.value
@@ -714,7 +723,6 @@ async fn update_flow(
));
}
let flow_path = flow_path.to_path();
let authed = maybe_refresh_folders(&flow_path, &w_id, authed, &db).await;
let mut tx = user_db.clone().begin(&authed).await?;
@@ -1037,6 +1045,7 @@ async fn get_flow_by_path(
Query(query): Query<WithStarredInfoQuery>,
) -> JsonResult<FlowWithStarred> {
let path = path.to_path();
check_scopes(&authed, || format!("flows:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let flow_o = if query.with_starred_info.unwrap_or(false) {
@@ -1104,10 +1113,11 @@ async fn get_flow_by_path_w_draft(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<FlowWDraft> {
let path = path.to_path();
check_scopes(&authed, || format!("flows:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let flow_o = sqlx::query_as::<_, FlowWDraft>(
"SELECT flow.path, flow.summary, flow,description, flow_version.schema, flow_version.value, flow.extra_perms, flow.draft_only, flow.ws_error_handler_muted, flow.dedicated_worker, draft.value as draft, flow.tag, flow.visible_to_runner_only, flow.on_behalf_of_email
"SELECT flow.path, flow.summary, flow.description, flow_version.schema, flow_version.value, flow.extra_perms, flow.draft_only, flow.ws_error_handler_muted, flow.dedicated_worker, draft.value as draft, flow.tag, flow.visible_to_runner_only, flow.on_behalf_of_email
FROM flow
LEFT JOIN draft
ON flow.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'flow'
@@ -1157,6 +1167,7 @@ async fn archive_flow_by_path(
Json(archived): Json<Archived>,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("flows:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
@@ -1233,6 +1244,7 @@ async fn delete_flow_by_path(
Query(query): Query<DeleteFlowQuery>,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("flows:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
@@ -1379,7 +1391,7 @@ mod tests {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}),
stop_after_if: Some(StopAfterIf {
expr: "foo = 'bar'".to_string(),
backend/windmill-api/src/folders.rs
+4 -3
View File
@@ -8,12 +8,11 @@
use std::sync::Arc;
use crate::db::ApiAuthed;
use crate::{
auth::AuthCache,
db::DB,
db::{ApiAuthed, DB},
users::Tokened,
utils::check_scopes,
webhook_util::{WebhookMessage, WebhookShared},
};
use axum::{
@@ -419,6 +418,7 @@ async fn get_folder(
Extension(user_db): Extension<UserDB>,
Path((w_id, name)): Path<(String, String)>,
) -> JsonResult<Folder> {
check_scopes(&authed, || format!("folders:read:f/{}", name))?;
let mut tx = user_db.begin(&authed).await?;
let folder = not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
@@ -457,6 +457,7 @@ async fn get_folder_usage(
Extension(user_db): Extension<UserDB>,
Path((w_id, name)): Path<(String, String)>,
) -> JsonResult<FolderUsage> {
check_scopes(&authed, || format!("folders:read:f/{}", name))?;
let mut tx = user_db.begin(&authed).await?;
let scripts = sqlx::query_scalar!(
backend/windmill-api/src/http_triggers.rs
+15 -7
View File
@@ -2,16 +2,17 @@
use crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs};
#[cfg(feature = "parquet")]
use crate::job_helpers_oss::get_workspace_s3_resource;
use crate::resources::try_get_resource_from_db_as;
use crate::trigger_helpers::{
get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, RunnableId,
};
use crate::utils::{non_empty_str, ExpiringCacheEntry};
use crate::{
auth::{AuthCache, OptTokened},
db::{ApiAuthed, DB},
resources::try_get_resource_from_db_as,
trigger_helpers::{
get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, RunnableId,
},
users::fetch_api_authed,
utils::{check_scopes, non_empty_str, ExpiringCacheEntry},
};
use anyhow::anyhow;
use axum::response::Response;
use axum::{
@@ -253,8 +254,9 @@ async fn get_trigger(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> error::JsonResult<HttpTrigger> {
let mut tx = user_db.begin(&authed).await?;
let path = path.to_path();
check_scopes(&authed, || format!("http_triggers:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let trigger = sqlx::query_as!(
HttpTrigger,
r#"
@@ -333,6 +335,10 @@ async fn create_trigger_inner(
new_http_trigger: &NewTrigger,
route_path_key: &str,
) -> WindmillResult<()> {
check_scopes(&authed, || {
format!("http_triggers:write:{}", &new_http_trigger.path)
})?;
sqlx::query!(
r#"
INSERT INTO http_trigger (
@@ -577,6 +583,7 @@ async fn update_trigger(
Json(ct): Json<EditTrigger>,
) -> WindmillResult<String> {
let path = path.to_path();
check_scopes(&authed, || format!("http_triggers:write:{}", path))?;
if *CLOUD_HOSTED && (ct.is_static_website || ct.static_asset_config.is_some()) {
return Err(error::Error::BadRequest(
@@ -761,8 +768,9 @@ async fn delete_trigger(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> WindmillResult<String> {
require_admin(authed.is_admin, &authed.username)?;
let path = path.to_path();
check_scopes(&authed, || format!("http_triggers:write:{}", path))?;
require_admin(authed.is_admin, &authed.username)?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"DELETE FROM http_trigger
backend/windmill-api/src/jobs.rs
+111 -70
View File
@@ -35,19 +35,15 @@ use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR};
use windmill_common::scripts::PREVIEW_IS_CODEBASE_HASH;
use windmill_common::variables::get_workspace_key;
use crate::add_webhook_allowed_origin;
use crate::auth::{OptTokened, Tokened};
use crate::concurrency_groups::join_concurrency_key;
use crate::db::ApiAuthed;
use crate::trigger_helpers::RunnableId;
use crate::users::get_scope_tags;
use crate::utils::content_plain;
use crate::{
add_webhook_allowed_origin,
args::{self, RawWebhookArgs},
db::DB,
users::{check_scopes, require_owner_of_path, OptAuthed},
utils::require_super_admin,
auth::{OptTokened, Tokened},
concurrency_groups::join_concurrency_key,
db::{ApiAuthed, DB},
trigger_helpers::RunnableId,
users::{get_scope_tags, require_owner_of_path, OptAuthed},
utils::{check_scopes, content_plain, require_super_admin},
};
use anyhow::Context;
use axum::{
@@ -722,7 +718,7 @@ macro_rules! get_job_query {
"SELECT \
{table}.id, {table}.workspace_id, parent_job, v2_job.created_by, v2_job.created_at, started_at, v2_job.runnable_id as script_hash, v2_job.runnable_path as script_path, \
CASE WHEN args is null THEN NULL
WHEN pg_column_size(args) < 90000 THEN
WHEN pg_column_size(args) < 90000 THEN
CASE WHEN jsonb_typeof(args) = 'object' THEN args
ELSE jsonb_build_object('value', args)
END
@@ -732,7 +728,7 @@ macro_rules! get_job_query {
{flow} as raw_flow, flow_step_id IS NOT NULL AS is_flow_step, script_lang as language, \
{lock} as raw_lock, permissioned_as_email as email, visible_to_owner, memory_peak as mem_peak, v2_job.tag, v2_job.priority, preprocessed, worker,\
{additional_fields} \
FROM {table}
FROM {table}
INNER JOIN v2_job ON v2_job.id = {table}.id \
{additional_joins} \
LEFT JOIN job_logs ON {table}.id = job_id \
@@ -1141,8 +1137,8 @@ async fn get_job_logs(
let record = sqlx::query!(
"SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index
FROM v2_as_completed_job
LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id
FROM v2_as_completed_job
LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id
WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))",
id,
w_id,
@@ -1185,8 +1181,8 @@ async fn get_job_logs(
} else {
let text = sqlx::query!(
"SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index
FROM v2_as_queue
LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id
FROM v2_as_queue
LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id
WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))",
id,
w_id,
@@ -1244,7 +1240,7 @@ async fn get_args(
.flatten();
let record = sqlx::query!(
"SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json<Box<RawValue>>\"
FROM v2_as_completed_job
FROM v2_as_completed_job
WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))",
id,
&w_id,
@@ -1784,7 +1780,6 @@ async fn list_filtered_job_uuids(
Query(lq): Query<ListCompletedQuery>,
) -> error::JsonResult<Vec<Uuid>> {
require_admin(authed.is_admin, &authed.username)?;
check_scopes(&authed, || format!("jobs:listjobs"))?;
let mut sqlb = list_completed_jobs_query(
w_id.as_str(),
@@ -1949,8 +1944,6 @@ async fn list_jobs(
Query(pagination): Query<Pagination>,
Query(lq): Query<ListCompletedQuery>,
) -> error::JsonResult<Vec<Job>> {
check_scopes(&authed, || format!("jobs:listjobs"))?;
let limit = pagination.per_page.unwrap_or(1000);
let (per_page, offset) = paginate(pagination);
let lqc = lq.clone();
@@ -2026,16 +2019,15 @@ pub async fn resume_suspended_flow_as_owner(
Path((_w_id, flow_id)): Path<(String, Uuid)>,
QueryOrBody(value): QueryOrBody<serde_json::Value>,
) -> error::Result<StatusCode> {
check_scopes(&authed, || format!("jobs:resumeflow"))?;
let value = value.unwrap_or(serde_json::Value::Null);
let mut tx = db.begin().await?;
let (flow, job_id) = get_suspended_flow_info(flow_id, &mut tx).await?;
require_owner_of_path(
&authed,
&flow.script_path.clone().unwrap_or_else(|| String::new()),
)?;
let flow_path = flow.script_path.as_deref().unwrap_or_else(|| "");
require_owner_of_path(&authed, flow_path)?;
check_scopes(&authed, || format!("jobs:run:flows:{}", flow_path))?;
let value = value.unwrap_or(serde_json::Value::Null);
insert_resume_job(
0,
@@ -3288,17 +3280,18 @@ async fn batch_rerun_jobs(
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(body): Json<BatchReRunJobsBodyArgs>,
) -> Response {
) -> error::Result<Response> {
check_scopes(&authed, || format!("jobs:run"))?;
let stream = batch_rerun_jobs_inner(authed, db, user_db, w_id, body);
let body = axum::body::Body::from_stream(stream.map(Result::<_, std::convert::Infallible>::Ok));
Response::builder()
Ok(Response::builder()
.status(201)
.header("Content-Type", "text/event-stream")
.header("Cache-Control", "no-cache")
.body(body)
.unwrap()
.unwrap())
}
fn batch_rerun_jobs_inner(
@@ -3312,7 +3305,7 @@ fn batch_rerun_jobs_inner(
tokio::spawn(async move {
let mut job_stream = sqlx::query_as!(
BatchReRunQueryReturnType,
r#"SELECT
r#"SELECT
j.id,
j.kind AS "kind: _",
COALESCE(s.path, f.path) AS "script_path!",
@@ -3507,8 +3500,9 @@ pub async fn run_flow_by_path_inner(
) -> error::Result<Uuid> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let flow_path = flow_path.to_path();
check_scopes(&authed, || format!("run:flow/{flow_path}"))?;
check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?;
let mut tx = user_db.clone().begin(&authed).await?;
@@ -3633,7 +3627,7 @@ pub async fn restart_flow(
let flow_path = completed_job
.script_path
.with_context(|| "No flow path set for completed flow job")?;
check_scopes(&authed, || format!("run:flow/{flow_path}"))?;
check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?;
let ehm = HashMap::new();
let push_args = completed_job
@@ -3713,8 +3707,7 @@ pub async fn run_script_by_path_inner(
check_license_key_valid().await?;
let script_path = script_path.to_path();
check_scopes(&authed, || format!("run:script/{script_path}"))?;
check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?;
let mut tx = user_db.clone().begin(&authed).await?;
let (job_payload, tag, delete_after_use, timeout, on_behalf_of) =
@@ -3787,6 +3780,11 @@ pub async fn run_workflow_as_code(
Query(wkflow_query): Query<WorkflowAsCodeQuery>,
Json(task): Json<WorkflowTask>,
) -> error::Result<(StatusCode, String)> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &authed).await?;
check_scopes(&authed, || format!("jobs:run"))?;
let mut i = 1;
if *CLOUD_HOSTED {
@@ -3794,10 +3792,6 @@ pub async fn run_workflow_as_code(
i += 1;
}
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &authed).await?;
if *CLOUD_HOSTED {
tracing::info!("workflow_as_code_tracing id {i} ");
i += 1;
@@ -3927,7 +3921,7 @@ pub async fn run_workflow_as_code(
VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3))
ON CONFLICT (id) DO UPDATE SET
workflow_as_code_status = JSONB_SET(
COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB),
COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB),
array[$2],
$3
)",
@@ -4364,6 +4358,9 @@ pub async fn run_wait_result_job_by_path_get(
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let script_path = script_path.to_path();
check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?;
if method == http::Method::HEAD {
return Ok(Json(serde_json::json!("")).into_response());
}
@@ -4380,8 +4377,6 @@ pub async fn run_wait_result_job_by_path_get(
let mut args = args.process_args(&authed, &db, &w_id, None).await?;
args.body = args::Body::HashMap(payload_args);
let script_path = script_path.to_path();
let args = args
.to_args_from_runnable(
&db,
@@ -4392,7 +4387,6 @@ pub async fn run_wait_result_job_by_path_get(
.await?;
check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
check_scopes(&authed, || format!("run:script/{script_path}"))?;
let mut tx = user_db.clone().begin(&authed).await?;
let (job_payload, tag, delete_after_use, timeout, on_behalf_authed) =
@@ -4531,9 +4525,10 @@ pub async fn run_wait_result_script_by_path_internal(
w_id: String,
args: PushArgsOwned,
) -> error::Result<Response> {
check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
let script_path = script_path.to_path();
check_scopes(&authed, || format!("run:script/{script_path}"))?;
check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?;
check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
let mut tx = user_db.clone().begin(&authed).await?;
let (job_payload, tag, delete_after_use, timeout, on_behalf_of) =
@@ -4640,7 +4635,7 @@ pub async fn run_wait_result_script_by_hash(
if let Some(run_query_cache_ttl) = run_query.cache_ttl {
cache_ttl = Some(run_query_cache_ttl);
}
check_scopes(&authed, || format!("run:script/{path}"))?;
check_scopes(&authed, || format!("jobs:run:scripts:{path}"))?;
let tag = run_query.tag.clone().or(tag);
check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?;
@@ -4746,7 +4741,7 @@ pub async fn run_wait_result_flow_by_path_internal(
check_queue_too_long(&db, run_query.queue_limit).await?;
let flow_path = flow_path.to_path();
check_scopes(&authed, || format!("run:flow/{flow_path}"))?;
check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?;
let scheduled_for = run_query.get_scheduled_for(&db).await?;
@@ -4830,8 +4825,6 @@ async fn run_preview_script(
) -> error::Result<(StatusCode, String)> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
check_scopes(&authed, || format!("jobs:runscript"))?;
if authed.is_operator {
return Err(error::Error::NotAuthorized(
"Operators cannot run preview jobs for security reasons".to_string(),
@@ -4901,7 +4894,6 @@ async fn run_bundle_preview_script(
) -> error::Result<(StatusCode, String)> {
use windmill_common::scripts::PREVIEW_IS_TAR_CODEBASE_HASH;
check_scopes(&authed, || format!("jobs:runscript"))?;
if authed.is_operator {
return Err(error::Error::NotAuthorized(
"Operators cannot run preview jobs for security reasons".to_string(),
@@ -5078,6 +5070,7 @@ async fn run_dependencies_job(
Path(w_id): Path<String>,
Json(req): Json<RunDependenciesRequest>,
) -> error::Result<Response> {
check_scopes(&authed, || format!("jobs:run"))?;
if authed.is_operator {
return Err(error::Error::NotAuthorized(
"Operators cannot run dependencies jobs for security reasons".to_string(),
@@ -5117,11 +5110,7 @@ async fn run_dependencies_job(
&db,
PushIsolationLevel::IsolatedRoot(db.clone()),
&w_id,
JobPayload::RawScriptDependencies {
script_path: script_path,
content: raw_code,
language: language,
},
JobPayload::RawScriptDependencies { script_path, content: raw_code, language },
args,
authed.display_username(),
&authed.email,
@@ -5153,6 +5142,7 @@ async fn run_dependencies_job(
pub struct RunFlowDependenciesRequest {
pub path: String,
pub flow_value: FlowValue,
pub raw_deps: Option<HashMap<String, String>>,
}
#[derive(Serialize)]
@@ -5166,21 +5156,27 @@ async fn run_flow_dependencies_job(
Path(w_id): Path<String>,
Json(req): Json<RunFlowDependenciesRequest>,
) -> error::Result<Response> {
check_scopes(&authed, || format!("jobs:run"))?;
if authed.is_operator {
return Err(error::Error::NotAuthorized(
"Operators cannot run dependencies jobs for security reasons".to_string(),
));
}
// Create args HashMap with skip_flow_update and raw_deps if present
let mut args_map = HashMap::from([("skip_flow_update".to_string(), to_raw_value(&true))]);
// Add raw_deps to args if present
if let Some(ref raw_deps) = req.raw_deps {
args_map.insert("raw_deps".to_string(), to_raw_value(raw_deps));
}
let (uuid, tx) = push(
&db,
PushIsolationLevel::IsolatedRoot(db.clone()),
&w_id,
JobPayload::RawFlowDependencies { path: req.path, flow_value: req.flow_value },
PushArgs::from(&HashMap::from([(
"skip_flow_update".to_string(),
to_raw_value(&true),
)])),
PushArgs::from(&args_map),
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
@@ -5311,10 +5307,10 @@ async fn add_batch_jobs(
} else if let Some(path) = batch_info.path {
let mut tx = user_db.clone().begin(&authed).await?;
let value_json = sqlx::query!(
"SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json<Box<RawValue>>\" FROM flow
"SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json<Box<RawValue>>\" FROM flow
LEFT JOIN flow_version
ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
LEFT JOIN flow_version_lite
LEFT JOIN flow_version_lite
ON flow_version_lite.id = flow_version.id
WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1",
&path, &w_id
@@ -5428,7 +5424,7 @@ async fn add_batch_jobs(
)
INSERT INTO v2_job_queue
(id, workspace_id, scheduled_for, tag)
(SELECT uuid, $1, $2, $3 FROM uuid_table)
(SELECT uuid, $1, $2, $3 FROM uuid_table)
RETURNING id"#,
w_id,
Utc::now(),
@@ -5446,7 +5442,7 @@ async fn add_batch_jobs(
.await?;
sqlx::query!(
"INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id)
"INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id)
SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8",
&uuids,
authed.email,
@@ -5473,7 +5469,7 @@ async fn add_batch_jobs(
if let Some(custom_concurrency_key) = custom_concurrency_key {
sqlx::query!(
"INSERT INTO concurrency_counter(concurrency_id, job_uuids)
"INSERT INTO concurrency_counter(concurrency_id, job_uuids)
VALUES ($1, '{}'::jsonb)",
&custom_concurrency_key
)
@@ -5501,7 +5497,6 @@ async fn run_preview_flow_job(
Query(run_query): Query<RunJobQuery>,
Json(raw_flow): Json<PreviewFlow>,
) -> error::Result<(StatusCode, String)> {
check_scopes(&authed, || format!("jobs:runflow"))?;
if authed.is_operator {
return Err(error::Error::NotAuthorized(
"Operators cannot run preview jobs for security reasons".to_string(),
@@ -5602,7 +5597,8 @@ pub async fn run_job_by_hash_inner(
delete_after_use,
..
} = get_script_info_for_hash(&mut *tx, &w_id, hash).await?;
check_scopes(&authed, || format!("run:script/{path}"))?;
check_scopes(&authed, || format!("jobs:run:scripts:{path}"))?;
if let Some(run_query_cache_ttl) = run_query.cache_ttl {
cache_ttl = Some(run_query_cache_ttl);
}
@@ -5750,6 +5746,7 @@ async fn get_job_update(
Path((w_id, job_id)): Path<(String, Uuid)>,
Query(JobUpdateQuery { log_offset, get_progress, running, only_result }): Query<JobUpdateQuery>,
) -> JsonResult<JobUpdate> {
<<<<<<< HEAD
Ok(Json(
get_job_update_data(
&opt_authed,
@@ -5767,6 +5764,54 @@ async fn get_job_update(
.await?,
))
}
=======
let record = sqlx::query!(
"SELECT
c.id IS NOT NULL AS completed,
CASE
WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)
ELSE false
END AS running,
SUBSTR(logs, GREATEST($1 - log_offset, 0)) AS logs,
COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,
CASE
-- flow step:
WHEN flow_step_id IS NOT NULL THEN NULL
-- completed:
WHEN c.id IS NOT NULL THEN COALESCE(
c.workflow_as_code_status || c.flow_status,
c.workflow_as_code_status,
c.flow_status
)
-- not completed:
ELSE COALESCE(
f.workflow_as_code_status || f.flow_status,
f.workflow_as_code_status,
f.flow_status
)
END AS \"flow_status: sqlx::types::Json<Box<RawValue>>\",
job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 AS log_offset,
created_by AS \"created_by!\",
CASE WHEN $4::BOOLEAN THEN (
SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'
) END AS progress
FROM v2_job j
LEFT JOIN v2_job_queue q USING (id)
LEFT JOIN v2_job_runtime r USING (id)
LEFT JOIN v2_job_status f USING (id)
LEFT JOIN v2_job_completed c USING (id)
LEFT JOIN job_logs ON job_logs.job_id = $3
WHERE j.workspace_id = $2 AND j.id = $3",
log_offset,
&w_id,
job_id,
get_progress.unwrap_or(false),
running,
)
.fetch_optional(&db)
.await?
.ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?;
>>>>>>> main
async fn get_job_update_sse(
OptAuthed(opt_authed): OptAuthed,
@@ -6276,8 +6321,6 @@ async fn list_completed_jobs(
Query(pagination): Query<Pagination>,
Query(lq): Query<ListCompletedQuery>,
) -> error::JsonResult<Vec<ListableCompletedJob>> {
check_scopes(&authed, || format!("jobs:listjobs"))?;
let (per_page, offset) = paginate(pagination);
let sql = list_completed_jobs_query(
@@ -6615,8 +6658,6 @@ async fn delete_completed_job<'a>(
Extension(db): Extension<DB>,
Path((w_id, id)): Path<(String, Uuid)>,
) -> error::Result<Response> {
check_scopes(&authed, || format!("jobs:deletejob"))?;
let mut tx = user_db.begin(&authed).await?;
require_admin(authed.is_admin, &authed.username)?;
backend/windmill-api/src/lib.rs
+7 -6
View File
@@ -34,11 +34,11 @@ use agent_workers_oss::AgentCache;
use anyhow::Context;
use argon2::Argon2;
use axum::extract::DefaultBodyLimit;
use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Router};
use axum::response::Response;
use axum::http::HeaderValue;
use axum::body::Body;
use axum::extract::DefaultBodyLimit;
use axum::http::HeaderValue;
use axum::response::Response;
use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Router};
use db::DB;
use reqwest::Client;
#[cfg(feature = "oauth2")]
@@ -153,6 +153,7 @@ mod schedule;
#[cfg(feature = "private")]
pub mod scim_ee;
mod scim_oss;
mod scopes;
mod scripts;
mod service_logs;
mod settings;
@@ -178,6 +179,7 @@ mod stripe_oss;
#[cfg(feature = "private")]
pub mod teams_ee;
mod teams_oss;
mod token;
mod tracing_init;
mod triggers;
mod users;
@@ -239,7 +241,6 @@ lazy_static::lazy_static! {
}
// Compliance with cloud events spec.
pub async fn add_webhook_allowed_origin(
req: axum::extract::Request,
@@ -262,7 +263,6 @@ pub async fn add_webhook_allowed_origin(
next.run(req).await
}
#[cfg(not(feature = "tantivy"))]
type IndexReader = ();
@@ -649,6 +649,7 @@ pub async fn run_server(
scim_oss::global_service()
.route_layer(axum::middleware::from_fn(has_scim_token)),
)
.nest("/tokens", token::global_service())
.nest("/concurrency_groups", concurrency_groups::global_service())
.nest("/scripts_u", scripts::global_unauthed_service())
.nest("/apps_u", {
backend/windmill-api/src/mqtt_triggers.rs
+13 -2
View File
@@ -4,6 +4,7 @@ use crate::{
resources::try_get_resource_from_db_as,
trigger_helpers::{trigger_runnable, TriggerJobArgs},
users::fetch_api_authed,
utils::check_scopes,
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
@@ -519,6 +520,9 @@ pub async fn create_mqtt_trigger(
Path(w_id): Path<String>,
Json(new_mqtt_trigger): Json<NewMqttTrigger>,
) -> error::Result<(StatusCode, String)> {
check_scopes(&authed, || {
format!("mqtt_triggers:write:{}", &new_mqtt_trigger.path)
})?;
if *CLOUD_HOSTED {
return Err(error::Error::BadRequest(
"MQTT triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(),
@@ -701,8 +705,10 @@ pub async fn get_mqtt_trigger(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<MqttTrigger> {
let mut tx = user_db.begin(&authed).await?;
let path = path.to_path();
check_scopes(&authed, || format!("mqtt_triggers:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let trigger = sqlx::query_as!(
MqttTrigger,
r#"
@@ -754,6 +760,8 @@ pub async fn update_mqtt_trigger(
Json(mqtt_trigger): Json<EditMqttTrigger>,
) -> error::Result<String> {
let workspace_path = path.to_path();
check_scopes(&authed, || format!("mqtt_triggers:write:{}", workspace_path))?;
let EditMqttTrigger {
mqtt_resource_path,
subscribe_topics,
@@ -856,6 +864,8 @@ pub async fn delete_mqtt_trigger(
Path((w_id, path)): Path<(String, StripPath)>,
) -> error::Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("mqtt_triggers:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
r#"
@@ -931,8 +941,9 @@ pub async fn set_enabled(
Path((w_id, path)): Path<(String, StripPath)>,
Json(payload): Json<SetEnabled>,
) -> error::Result<String> {
let mut tx = user_db.begin(&authed).await?;
let path = path.to_path();
check_scopes(&authed, || format!("mqtt_triggers:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
// important to set server_id, last_server_ping and error to NULL to stop current mqtt listener
let one_o = sqlx::query_scalar!(
backend/windmill-api/src/postgres_triggers/handler.rs
+9 -2
View File
@@ -6,6 +6,7 @@ use std::collections::{
use crate::{
db::{ApiAuthed, DB},
postgres_triggers::mapper::{Mapper, MappingInfo},
utils::check_scopes,
};
use axum::{
extract::{Path, Query},
@@ -373,6 +374,8 @@ pub async fn create_postgres_trigger(
Path(w_id): Path<String>,
Json(new_postgres_trigger): Json<NewPostgresTrigger>,
) -> Result<(StatusCode, String)> {
check_scopes(&authed, || format!("postgres_triggers:write:{}", new_postgres_trigger.path))?;
if *CLOUD_HOSTED {
return Err(error::Error::BadRequest(
"Postgres triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(),
@@ -1150,8 +1153,9 @@ pub async fn get_postgres_trigger(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<PostgresTrigger> {
let mut tx = user_db.begin(&authed).await?;
let path = path.to_path();
check_scopes(&authed, || format!("postgres_triggers:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let trigger = sqlx::query_as!(
PostgresTrigger,
r#"
@@ -1200,6 +1204,7 @@ pub async fn update_postgres_trigger(
Json(postgres_trigger): Json<EditPostgresTrigger>,
) -> Result<String> {
let workspace_path = path.to_path();
check_scopes(&authed, || format!("postgres_triggers:write:{}", workspace_path))?;
let EditPostgresTrigger {
replication_slot_name,
@@ -1332,6 +1337,7 @@ pub async fn delete_postgres_trigger(
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("postgres_triggers:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
r#"
@@ -1403,8 +1409,9 @@ pub async fn set_enabled(
Path((w_id, path)): Path<(String, StripPath)>,
Json(payload): Json<SetEnabled>,
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
let path = path.to_path();
check_scopes(&authed, || format!("postgres_triggers:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
// important to set server_id, last_server_ping and error to NULL to stop current postgres listener
let one_o = sqlx::query_scalar!(
backend/windmill-api/src/raw_apps.rs
+6
View File
@@ -8,6 +8,7 @@
use crate::{
db::{ApiAuthed, DB},
users::require_owner_of_path,
utils::check_scopes,
webhook_util::{WebhookMessage, WebhookShared},
};
use axum::{
@@ -128,6 +129,7 @@ async fn get_data(
Path((w_id, _version, path)): Path<(String, u16, StripPath)>,
) -> Result<Response> {
let path = path.to_path();
check_scopes(&authed, || format!("raw_apps:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let app_o = sqlx::query_scalar!(
@@ -154,6 +156,7 @@ async fn create_app(
Path(w_id): Path<String>,
Json(app): Json<CreateApp>,
) -> Result<(StatusCode, String)> {
check_scopes(&authed, || format!("raw_apps:write:{}", app.path))?;
if *CLOUD_HOSTED {
let nb_apps = sqlx::query_scalar!(
"SELECT COUNT(*) FROM raw_app WHERE workspace_id = $1",
@@ -233,6 +236,7 @@ async fn delete_app(
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("raw_apps:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
@@ -271,6 +275,7 @@ async fn update_app(
use sql_builder::prelude::*;
let path = path.to_path();
check_scopes(&authed, || format!("raw_apps:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let mut sqlb = SqlBuilder::update_table("raw_app");
@@ -347,6 +352,7 @@ async fn exists_app(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<bool> {
let path = path.to_path();
// Note: exists_app doesn't require authentication, so no scope check needed
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
path,
backend/windmill-api/src/resources.rs
+11 -1
View File
@@ -11,6 +11,7 @@ use std::collections::HashMap;
use crate::{
db::{ApiAuthed, DB},
users::{maybe_refresh_folders, require_owner_of_path, Tokened},
utils::check_scopes,
webhook_util::{WebhookMessage, WebhookShared},
};
use axum::{
@@ -280,6 +281,7 @@ async fn get_resource(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<ListableResource> {
let path = path.to_path();
check_scopes(&authed, || format!("resources:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let resource_o = sqlx::query_as!(
@@ -331,6 +333,7 @@ async fn get_resource_value(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Option<serde_json::Value>> {
let path = path.to_path();
check_scopes(&authed, || format!("resources:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let value_o = sqlx::query_scalar!(
@@ -422,12 +425,15 @@ async fn get_resource_value_interpolated(
Path((w_id, path)): Path<(String, StripPath)>,
Query(job_info): Query<JobInfo>,
) -> JsonResult<Option<serde_json::Value>> {
let path = path.to_path();
check_scopes(&authed, || format!("resources:read:{}", path))?;
return get_resource_value_interpolated_internal(
&authed,
Some(user_db),
&db,
w_id.as_str(),
path.to_path(),
path,
job_info.job_id,
token.as_str(),
)
@@ -658,6 +664,7 @@ async fn create_resource(
Query(q): Query<CreateResourceQuery>,
Json(resource): Json<CreateResource>,
) -> Result<(StatusCode, String)> {
check_scopes(&authed, || format!("resources:write:{}", resource.path))?;
if *CLOUD_HOSTED {
let nb_resources = sqlx::query_scalar!(
"SELECT COUNT(*) FROM resource WHERE workspace_id = $1",
@@ -740,6 +747,7 @@ async fn delete_resource(
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("resources:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
@@ -798,6 +806,7 @@ async fn update_resource(
use sql_builder::prelude::*;
let path = path.to_path();
check_scopes(&authed, || format!("resources:write:{}", path))?;
let mut sqlb = SqlBuilder::update_table("resource");
sqlb.and_where_eq("path", "?".bind(&path));
@@ -891,6 +900,7 @@ async fn update_resource_value(
Json(nv): Json<UpdateResource>,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("resources:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
backend/windmill-api/src/schedule.rs
+11 -2
View File
@@ -7,7 +7,10 @@
*/
use crate::{
db::{ApiAuthed, DB}, settings::{delete_global_setting, set_global_setting_internal}, users::maybe_refresh_folders, utils::require_super_admin
db::{ApiAuthed, DB},
settings::{delete_global_setting, set_global_setting_internal},
users::maybe_refresh_folders,
utils::{check_scopes, require_super_admin},
};
use axum::{
extract::{Extension, Path, Query},
@@ -128,6 +131,8 @@ async fn create_schedule(
Path(w_id): Path<String>,
Json(ns): Json<NewSchedule>,
) -> Result<String> {
check_scopes(&authed, || format!("schedules:write:{}", ns.path))?;
let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
#[cfg(not(feature = "enterprise"))]
@@ -294,6 +299,7 @@ async fn edit_schedule(
Json(es): Json<EditSchedule>,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("schedules:write:{}", path))?;
let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await;
let mut tx = user_db.begin(&authed).await?;
@@ -560,6 +566,7 @@ async fn get_schedule(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Schedule> {
let path = path.to_path();
check_scopes(&authed, || format!("schedules:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let schedule_o = windmill_queue::schedule::get_schedule_opt(&mut *tx, &w_id, path).await?;
@@ -608,6 +615,7 @@ pub async fn set_enabled(
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
let path = path.to_path();
check_scopes(&authed, || format!("schedules:write:{}", path))?;
let schedule_o = sqlx::query_as!(
Schedule,
r#"
@@ -745,8 +753,9 @@ async fn delete_schedule(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
let path = path.to_path();
check_scopes(&authed, || format!("schedules:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
clear_schedule(&mut tx, path, &w_id).await?;
let exists = sqlx::query_scalar!(
backend/windmill-api/src/scopes.rs
+868
View File
@@ -0,0 +1,868 @@
/*
* Author: Windmill Labs, Inc
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use windmill_common::error::{Error, Result};
/// Comprehensive scope system for JWT token authorization
///
/// Scopes follow the format: {domain}:{action}[:{resource}]
/// Examples:
/// - "jobs:read" - Read access to jobs
/// - "scripts:write:f/folder/*" - Write access to scripts in a folder
/// - "*" - Full access (superuser)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScopeDefinition {
pub domain: String,
pub action: String,
pub kind: Option<String>, // For jobs:run:kind (optional)
pub resource: Option<Vec<String>>,
}
impl ScopeDefinition {
pub fn new(
domain: &str,
action: &str,
kind: Option<&str>,
resource: Option<Vec<String>>,
) -> Self {
Self {
domain: domain.to_string(),
action: action.to_string(),
kind: kind.map(|s| s.to_string()),
resource: resource,
}
}
pub fn from_scope_string(scope: &str) -> Result<Self> {
let parts: Vec<&str> = scope.split(':').collect();
let into_owned_vec = |resources: &str| -> Vec<String> {
let resources = resources
.split(",")
.collect_vec()
.into_iter()
.map(ToOwned::to_owned)
.collect_vec();
resources
};
match parts.len() {
2 => Ok(Self::new(parts[0], parts[1], None, None)), // domain:action
3 => {
if parts[0] == "jobs" && parts[1] == "run" {
Ok(Self::new(parts[0], parts[1], Some(parts[2]), None))
} else {
Ok(Self::new(
parts[0],
parts[1],
None,
Some(into_owned_vec(parts[2])),
))
}
}
4 => {
if parts[0] == "jobs" && parts[1] == "run" {
Ok(Self::new(
parts[0],
parts[1],
Some(parts[2]),
Some(into_owned_vec(parts[3])),
))
} else {
Err(Error::BadRequest(format!(
"Invalid 4-part scope: {}",
scope
)))
}
}
_ => Err(Error::BadRequest(format!(
"Invalid scope format: {}",
scope
))),
}
}
pub fn as_string(&self) -> String {
match (&self.kind, &self.resource) {
(Some(kind), Some(resource)) => {
format!(
"{}:{}:{}:{}",
self.domain,
self.action,
kind,
resource.join(",")
)
}
(Some(kind), None) => {
format!("{}:{}:{}", self.domain, self.action, kind)
}
(None, Some(resource)) => {
format!("{}:{}:{}", self.domain, self.action, resource.join(","))
}
(None, None) => format!("{}:{}", self.domain, self.action),
}
}
pub fn includes(&self, other: &ScopeDefinition) -> bool {
if self.domain != other.domain {
return false;
}
match (self.action.as_str(), other.action.as_str()) {
(a, b) if (a == "write" && b == "read") || (a == b) => {}
_ => return false,
}
if self.domain == "jobs" && self.action == "run" {
match (&self.kind, &other.kind) {
(Some(self_kind), Some(other_kind)) => {
if self_kind != other_kind {
return false;
}
}
(Some(_), None) => {
return false;
}
(None, _) => {
return true;
}
}
}
match (&self.resource, &other.resource) {
(Some(self_resources), Some(other_resources)) => {
resources_match(self_resources, other_resources)
}
(Some(_), None) => false,
(None, _) => true,
}
}
}
fn resources_match(scope_resources: &[String], accepted_resources: &[String]) -> bool {
if scope_resources.contains(&"*".to_string()) || accepted_resources.contains(&"*".to_string()) {
return true;
}
if scope_resources.len() <= 4 && accepted_resources.len() <= 4 {
return resources_match_small(scope_resources, accepted_resources);
}
resources_match_large(scope_resources, accepted_resources)
}
fn resources_match_small(scope_resources: &[String], accepted_resources: &[String]) -> bool {
for required in accepted_resources {
for scope_resource in scope_resources {
if resource_matches_pattern(scope_resource, required) {
return true;
}
}
}
false
}
fn resources_match_large(scope_resources: &[String], accepted_resources: &[String]) -> bool {
let mut exact_matches = HashSet::new();
let mut patterns = Vec::new();
for scope_resource in scope_resources {
if scope_resource.contains('*') {
patterns.push(scope_resource);
} else {
exact_matches.insert(scope_resource);
}
}
for accepted_resource in accepted_resources {
if exact_matches.contains(accepted_resource) {
return true;
}
for pattern in &patterns {
if resource_matches_pattern(pattern, accepted_resource) {
return true;
}
}
}
false
}
fn resource_matches_pattern(scope_resource: &str, accepted_resource: &str) -> bool {
if scope_resource == accepted_resource {
return true;
}
let matches_wildcard = |pattern: &str, resource: &str| -> bool {
if !pattern.ends_with("/*") {
return false;
}
let prefix = &pattern[..pattern.len() - 2];
if !resource.starts_with(prefix) {
return false;
}
// If the resource is exactly the prefix, it matches
if resource.len() == prefix.len() {
return true;
}
// If the resource is longer, the next character must be '/' for a valid match
// This prevents "u/user" from matching "u/use/*"
resource.chars().nth(prefix.len()) == Some('/')
};
// Check if either resource is a wildcard pattern and matches the other
matches_wildcard(scope_resource, accepted_resource)
|| matches_wildcard(accepted_resource, scope_resource)
}
/// Available scope domains (top-level API categories)
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ScopeDomain {
// Core resource domains
Jobs,
Scripts,
Flows,
Apps,
Variables,
Resources,
Schedules,
Folders,
Users,
Groups,
Workspaces,
// Trigger domains
HttpTriggers,
WebsocketTriggers,
KafkaTriggers,
NatsTriggers,
MqttTriggers,
SqsTriggers,
GcpTriggers,
PostgresTriggers,
// System domains
Audit,
Settings,
Workers,
ServiceLogs,
Configs,
OAuth,
AI,
Indexer,
Teams, // Microsoft Teams integration
GitSync, // Git synchronization
// Special domains
Capture, // Webhook capture
Drafts, // Draft resources
Favorites, // User favorites
Inputs, // Input templates
JobHelpers, // Job helper functions
ConcurrencyGroups, // Concurrency groups
Oidc, // OpenID Connect
Openapi, // OpenAPI generation
// Additional domains
Acls, // Granular access control lists
RawApps, // Raw application data
AgentWorkers, // Agent workers management
}
impl ScopeDomain {
pub fn as_str(&self) -> &'static str {
match self {
Self::Jobs => "jobs",
Self::Scripts => "scripts",
Self::Flows => "flows",
Self::Apps => "apps",
Self::Variables => "variables",
Self::Resources => "resources",
Self::Schedules => "schedules",
Self::Folders => "folders",
Self::Users => "users",
Self::Groups => "groups",
Self::Workspaces => "workspaces",
Self::HttpTriggers => "http_triggers",
Self::WebsocketTriggers => "websocket_triggers",
Self::KafkaTriggers => "kafka_triggers",
Self::NatsTriggers => "nats_triggers",
Self::MqttTriggers => "mqtt_triggers",
Self::SqsTriggers => "sqs_triggers",
Self::GcpTriggers => "gcp_triggers",
Self::PostgresTriggers => "postgres_triggers",
Self::Audit => "audit",
Self::Settings => "settings",
Self::Workers => "workers",
Self::ServiceLogs => "service_logs",
Self::Configs => "configs",
Self::OAuth => "oauth",
Self::AI => "ai",
Self::Capture => "capture",
Self::Drafts => "drafts",
Self::Favorites => "favorites",
Self::Inputs => "inputs",
Self::JobHelpers => "job_helpers",
Self::ConcurrencyGroups => "concurrency_groups",
Self::Oidc => "oidc",
Self::Openapi => "openapi",
Self::Acls => "acls",
Self::RawApps => "raw_apps",
Self::AgentWorkers => "agent_workers",
Self::Indexer => "indexer",
Self::Teams => "teams",
Self::GitSync => "git_sync",
}
}
pub fn from_str(s: &str) -> Option<Self> {
match s {
"jobs" | "jobs_u" => Some(Self::Jobs),
"scripts" => Some(Self::Scripts),
"flows" => Some(Self::Flows),
"apps" => Some(Self::Apps),
"variables" => Some(Self::Variables),
"resources" => Some(Self::Resources),
"schedules" => Some(Self::Schedules),
"folders" => Some(Self::Folders),
"users" => Some(Self::Users),
"groups" => Some(Self::Groups),
"workspaces" => Some(Self::Workspaces),
"http_triggers" => Some(Self::HttpTriggers),
"websocket_triggers" => Some(Self::WebsocketTriggers),
"kafka_triggers" => Some(Self::KafkaTriggers),
"nats_triggers" => Some(Self::NatsTriggers),
"mqtt_triggers" => Some(Self::MqttTriggers),
"sqs_triggers" => Some(Self::SqsTriggers),
"gcp_triggers" => Some(Self::GcpTriggers),
"postgres_triggers" => Some(Self::PostgresTriggers),
"audit" => Some(Self::Audit),
"settings" => Some(Self::Settings),
"workers" => Some(Self::Workers),
"service_logs" => Some(Self::ServiceLogs),
"configs" => Some(Self::Configs),
"oauth" => Some(Self::OAuth),
"ai" => Some(Self::AI),
"indexer" | "srch" => Some(Self::Indexer),
"teams" => Some(Self::Teams),
"git_sync" | "github_app" => Some(Self::GitSync),
"capture" => Some(Self::Capture),
"drafts" => Some(Self::Drafts),
"favorites" => Some(Self::Favorites),
"inputs" => Some(Self::Inputs),
"job_helpers" => Some(Self::JobHelpers),
"concurrency_groups" => Some(Self::ConcurrencyGroups),
"oidc" => Some(Self::Oidc),
"openapi" => Some(Self::Openapi),
"acls" => Some(Self::Acls),
"raw_apps" => Some(Self::RawApps),
"agent_workers" => Some(Self::AgentWorkers),
_ => None,
}
}
}
/// Available scope actions
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ScopeAction {
Read, // GET operations, list, view
Write, // POST, PUT, PATCH, DELETE operations, create, update, delete
Run, // Special action for running (scripts, flows, etc.)
}
impl ScopeAction {
pub fn as_str(&self) -> &'static str {
match self {
Self::Read => "read",
Self::Write => "write",
Self::Run => "run",
}
}
pub fn from_str(s: &str) -> Option<Self> {
match s {
"read" => Some(Self::Read),
"write" => Some(Self::Write),
"delete" => Some(Self::Write),
"run" => Some(Self::Run),
_ => None,
}
}
/// Check if this action includes another action
/// Write includes Read
pub fn includes(&self, other: &ScopeAction) -> bool {
match (self, other) {
(ScopeAction::Write, ScopeAction::Read) => true,
(a, b) => a == b,
}
}
}
pub fn check_route_access(
token_scopes: &[String],
route_path: &str,
http_method: &str,
) -> Result<()> {
// Map HTTP method to scope action (considering route context)
let required_action = map_http_method_to_action(http_method, route_path);
// Find the domain and kind for this route
let (required_domain, required_kind) = extract_domain_from_route(route_path)?;
let mut is_scoped_token = false;
// Check if any token scope grants the required access
for scope_str in token_scopes {
if !scope_str.starts_with("if_jobs:filter_tags:") {
if let Ok(scope) = ScopeDefinition::from_scope_string(scope_str) {
if scope_grants_access(
&scope,
required_domain,
required_action,
required_kind.as_deref(),
)? {
return Ok(());
}
}
if !is_scoped_token {
is_scoped_token = true;
}
}
}
//Edge case for backward compatibility, if only scopes defined was filter tag then don't treat this we don't treat the token
//as a restricted token
if !is_scoped_token {
return Ok(());
}
let scope_display = if let Some(kind) = required_kind {
format!(
"{}:{}:{}",
required_domain.as_str(),
required_action.as_str(),
kind
)
} else {
format!("{}:{}", required_domain.as_str(), required_action.as_str())
};
Err(Error::NotAuthorized(format!(
"Access denied. Required scope: {}",
scope_display
)))
}
const SCRIPT_JOBS: [&'static str; 6] = [
"jobs/run/p",
"jobs/run/h",
"jobs/run_wait_result/p",
"jobs/run_wait_result/h",
"jobs/run/preview_bundle",
"jobs/run/preview",
];
const FLOW_JOBS: [&'static str; 5] = [
"jobs/run/f",
"jobs/run_wait_result/f",
"jobs/run/preview_flow",
"jobs/restart/f",
"jobs/flow/resume",
];
lazy_static::lazy_static! {
static ref RUN_PATH_ACTIONS: Vec<&'static str> = {
let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies",];
v.extend(SCRIPT_JOBS);
v.extend(FLOW_JOBS);
v
};
}
fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction {
if RUN_PATH_ACTIONS
.iter()
.any(|run_path| route_path.contains(run_path))
{
return ScopeAction::Run;
}
match method.to_uppercase().as_str() {
"GET" | "HEAD" | "OPTIONS" => ScopeAction::Read,
"POST" | "PUT" | "PATCH" | "DELETE" => ScopeAction::Write,
_ => ScopeAction::Read,
}
}
/// Checks the route path to determine the runnable kind (either "flows" or "scripts").
///
/// The order of checks is important:
/// - Flow-related paths are checked first to avoid false positives, as some flow paths
/// (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`).
///
/// Returns `"flows"` or `"scripts"` based on the match, or `None` if no match is found.
fn determine_kind_from_route(route_path: &str) -> Option<String> {
if route_path.starts_with("jobs") {
if FLOW_JOBS.iter().any(|path| route_path.starts_with(path)) {
return Some("flows".to_string());
} else if SCRIPT_JOBS.iter().any(|path| route_path.starts_with(path)) {
return Some("scripts".to_string());
}
}
None
}
fn extract_domain_from_route(route_path: &str) -> Result<(ScopeDomain, Option<String>)> {
// Examples:
// - /api/w/workspace/jobs/123 -> jobs domain (workspaced)
// - /api/teams/sync -> teams domain (global)
// - /api/srch/index/search -> indexer domain (global)
let parts: Vec<&str> = route_path.split('/').collect();
let (domain, kind) = if parts.len() >= 5 && parts[1] == "api" && parts[2] == "w" {
let domain_part = parts[4];
let route_suffix = &parts[4..].join("/");
let domain = ScopeDomain::from_str(domain_part);
let kind = determine_kind_from_route(route_suffix);
(domain, kind)
} else if parts.len() >= 3 && parts[1] == "api" {
(ScopeDomain::from_str(parts[2]), None)
} else {
(None, None)
};
if let Some(domain) = domain {
return Ok((domain, kind));
}
Err(Error::BadRequest(format!(
"Could not extract domain from route: {}",
route_path
)))
}
fn scope_grants_access(
scope: &ScopeDefinition,
required_domain: ScopeDomain,
required_action: ScopeAction,
required_kind: Option<&str>,
) -> Result<bool> {
// Check domain match
let scope_domain = ScopeDomain::from_str(&scope.domain)
.ok_or_else(|| Error::BadRequest(format!("Invalid scope domain: {}", scope.domain)))?;
if scope_domain != required_domain {
return Ok(false);
}
// Check action match (with hierarchical permissions)
let scope_action = ScopeAction::from_str(&scope.action)
.ok_or_else(|| Error::BadRequest(format!("Invalid scope action: {}", scope.action)))?;
if !scope_action.includes(&required_action) {
return Ok(false);
}
if required_domain == ScopeDomain::Jobs && required_action == ScopeAction::Run {
match (&scope.kind, required_kind) {
(Some(scope_kind), Some(req_kind)) => {
if scope_kind != req_kind {
return Ok(false);
}
}
(None, _) => {}
(Some(_), None) => {
return Ok(false);
}
}
}
// No resource specified means access to entire domain
Ok(true)
}
/// Helper function to check if scopes allow access to a route
pub fn check_scopes_for_route(
token_scopes: Option<&[String]>,
route_path: &str,
http_method: &str,
) -> Result<()> {
// If no scopes defined, allow access (backward compatibility)
let scopes = match token_scopes {
Some(s) if !s.is_empty() => s,
_ => return Ok(()),
};
check_route_access(scopes, route_path, http_method)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_scope_definition_parsing() {
let scope = ScopeDefinition::from_scope_string("jobs:read").unwrap();
assert_eq!(scope.domain, "jobs");
assert_eq!(scope.action, "read");
assert_eq!(scope.kind, None);
assert_eq!(scope.resource, None);
let scope = ScopeDefinition::from_scope_string("jobs:run:scripts:f/folder/*").unwrap();
assert_eq!(scope.domain, "jobs");
assert_eq!(scope.action, "run");
assert_eq!(scope.kind, Some("scripts".to_string()));
assert_eq!(scope.resource, Some(vec!["f/folder/*".to_string()]));
// Test jobs:run:kind parsing
let scope = ScopeDefinition::from_scope_string("jobs:run:scripts").unwrap();
assert_eq!(scope.domain, "jobs");
assert_eq!(scope.action, "run");
assert_eq!(scope.kind, Some("scripts".to_string()));
assert_eq!(scope.resource, None);
// Test jobs:run:kind:resource parsing
let scope = ScopeDefinition::from_scope_string("jobs:run:flows:f/folder/*").unwrap();
assert_eq!(scope.domain, "jobs");
assert_eq!(scope.action, "run");
assert_eq!(scope.kind, Some("flows".to_string()));
assert_eq!(scope.resource, Some(vec!["f/folder/*".to_string()]));
// Test comma-separated resources parsing
let scope =
ScopeDefinition::from_scope_string("scripts:read:path1,path2,f/folder/*").unwrap();
assert_eq!(scope.domain, "scripts");
assert_eq!(scope.action, "read");
assert_eq!(scope.kind, None);
assert_eq!(
scope.resource,
Some(vec![
"path1".to_string(),
"path2".to_string(),
"f/folder/*".to_string()
])
);
}
#[test]
fn test_scope_action_hierarchy() {
assert!(ScopeAction::Write.includes(&ScopeAction::Read));
assert!(!ScopeAction::Read.includes(&ScopeAction::Write));
assert!(!ScopeAction::Run.includes(&ScopeAction::Read));
assert!(!ScopeAction::Run.includes(&ScopeAction::Write));
}
#[test]
fn test_route_domain_extraction() {
let (domain, kind) = extract_domain_from_route("/api/w/test_workspace/jobs/123").unwrap();
assert_eq!(domain, ScopeDomain::Jobs);
assert_eq!(kind, None);
let (domain, kind) =
extract_domain_from_route("/api/w/test_workspace/scripts/test_script").unwrap();
assert_eq!(domain, ScopeDomain::Scripts);
assert_eq!(kind, None);
}
#[test]
fn test_specific_scope_access() {
let scopes = vec!["jobs:read".to_string()];
assert!(check_route_access(&scopes, "/api/w/test_workspace/jobs/123", "GET").is_ok());
// DELETE now requires write permission, so it should still fail with read-only scope
assert!(check_route_access(&scopes, "/api/w/test_workspace/jobs/123", "DELETE").is_err());
}
#[test]
fn test_new_domain_parsing() {
// Test that new domains are properly parsed
assert_eq!(ScopeDomain::from_str("acls"), Some(ScopeDomain::Acls));
assert_eq!(
ScopeDomain::from_str("raw_apps"),
Some(ScopeDomain::RawApps)
);
assert_eq!(
ScopeDomain::from_str("agent_workers"),
Some(ScopeDomain::AgentWorkers)
);
// Test that string conversion works both ways
assert_eq!(ScopeDomain::Acls.as_str(), "acls");
assert_eq!(ScopeDomain::RawApps.as_str(), "raw_apps");
assert_eq!(ScopeDomain::AgentWorkers.as_str(), "agent_workers");
}
#[test]
fn test_resource_array_matching() {
// Test wildcard access
let scope_all = ScopeDefinition::new("scripts", "read", None, Some(vec!["*".to_string()]));
let required = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["path1".to_string(), "path2".to_string()]),
);
assert!(scope_all.includes(&required));
// Test exact matches
let scope_exact = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["path1".to_string(), "path2".to_string()]),
);
let required_subset =
ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()]));
assert!(scope_exact.includes(&required_subset));
// Test partial match - should grant access if ANY required resource matches
let scope_limited =
ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()]));
let required_partial = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["path1".to_string(), "path2".to_string()]),
);
assert!(scope_limited.includes(&required_partial)); // path1 matches, so access granted
// Test no match - scope doesn't cover any of the required resources
let scope_different =
ScopeDefinition::new("scripts", "read", None, Some(vec!["path3".to_string()]));
let required_no_match = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["path1".to_string(), "path2".to_string()]),
);
assert!(!scope_different.includes(&required_no_match));
// Test pattern matching
let scope_pattern = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["f/folder/*".to_string()]),
);
let required_in_folder = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["f/folder/script1".to_string()]),
);
assert!(scope_pattern.includes(&required_in_folder));
// Test mixed patterns and exact matches
let scope_mixed = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["exact_path".to_string(), "f/folder/*".to_string()]),
);
let required_mixed1 = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["exact_path".to_string()]),
);
let required_mixed2 = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["f/folder/script2".to_string()]),
);
assert!(scope_mixed.includes(&required_mixed1));
assert!(scope_mixed.includes(&required_mixed2));
}
#[test]
fn test_efficiency_small_vs_large_arrays() {
// Test small array optimization path
let scope_small = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["path1".to_string(), "path2".to_string()]),
);
let required_small =
ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()]));
assert!(scope_small.includes(&required_small));
// Test large array optimization path
let large_scope_vec: Vec<String> = (0..10).map(|i| format!("path{}", i)).collect();
let scope_large = ScopeDefinition::new("scripts", "read", None, Some(large_scope_vec));
let required_large =
ScopeDefinition::new("scripts", "read", None, Some(vec!["path5".to_string()]));
assert!(scope_large.includes(&required_large));
}
#[test]
fn test_user_example_case() {
// User's example: scope has "u/dieri/*", required has ["u/dadad/wqdq", "u/*"]
// Should grant access because scope "u/dieri/*" falls under required pattern "u/*"
let user_scope =
ScopeDefinition::new("scripts", "read", None, Some(vec!["u/dieri/*".to_string()]));
let required_mixed = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["u/dadad/wqdq".to_string(), "u/*".to_string()]),
);
assert!(user_scope.includes(&required_mixed)); // Should match because u/dieri/* falls under u/*
// Another example: scope covers one but not both paths
let scope_specific = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["folder/file1".to_string()]),
);
let required_multi = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["folder/file1".to_string(), "other/file2".to_string()]),
);
assert!(scope_specific.includes(&required_multi)); // Should match because folder/file1 matches exactly
// Test bidirectional pattern matching more explicitly
let scope_broad =
ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()]));
let required_specific = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["u/dieri/script.py".to_string()]),
);
assert!(scope_broad.includes(&required_specific)); // u/* covers u/dieri/script.py
let scope_specific_path = ScopeDefinition::new(
"scripts",
"read",
None,
Some(vec!["u/dieri/script.py".to_string()]),
);
let required_broad =
ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()]));
assert!(scope_specific_path.includes(&required_broad)); // u/dieri/script.py satisfies u/*
}
}
backend/windmill-api/src/scripts.rs
+33 -18
View File
@@ -14,7 +14,7 @@ use crate::{
get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail,
},
users::{maybe_refresh_folders, require_owner_of_path},
utils::WithStarredInfoQuery,
utils::{check_scopes, WithStarredInfoQuery},
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
};
@@ -43,7 +43,7 @@ use windmill_audit::ActionKind;
use windmill_worker::process_relative_imports;
use windmill_common::{
assets::{clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind},
assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType},
error::to_anyhow,
worker::CLOUD_HOSTED,
};
@@ -116,6 +116,9 @@ pub struct ScriptWDraft {
pub has_preprocessor: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub on_behalf_of_email: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[sqlx(json(nullable))]
pub assets: Option<Vec<AssetWithAltAccessType>>,
}
pub fn global_service() -> Router {
@@ -516,6 +519,8 @@ async fn create_script_internal<'c>(
user_db: UserDB,
webhook: WebhookShared,
) -> Result<(ScriptHash, Transaction<'c, Postgres>)> {
check_scopes(&authed, || format!("scripts:write:{}", ns.path))?;
let codebase = ns.codebase.as_ref();
#[cfg(not(feature = "enterprise"))]
if ns.ws_error_handler_muted.is_some_and(|val| val) {
@@ -769,8 +774,8 @@ async fn create_script_internal<'c>(
content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \
draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \
dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \
delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33)",
delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)",
&w_id,
&hash.0,
ns.path,
@@ -808,6 +813,7 @@ async fn create_script_internal<'c>(
None
},
validate_schema,
ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok())
)
.execute(&mut *tx)
.await?;
@@ -922,16 +928,8 @@ async fn create_script_internal<'c>(
}
clear_asset_usage(&mut *tx, &w_id, &script_path, AssetUsageKind::Script).await?;
for asset in parse_assets(&ns.content, ns.language)?.iter().flatten() {
insert_asset_usage(
&mut *tx,
&w_id,
asset,
ns.fallback_access_types.as_ref().map(Vec::as_slice),
&ns.path,
AssetUsageKind::Script,
)
.await?;
for asset in ns.assets.as_ref().into_iter().flatten() {
insert_asset_usage(&mut *tx, &w_id, &asset, &ns.path, AssetUsageKind::Script).await?;
}
let permissioned_as = username_to_permissioned_as(&authed.username);
@@ -1062,6 +1060,7 @@ async fn get_script_by_path(
Query(query): Query<WithStarredInfoQuery>,
) -> JsonResult<ScriptWithStarred> {
let path = path.to_path();
check_scopes(&authed, || format!("scripts:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let script_o = if query.with_starred_info.unwrap_or(false) {
@@ -1119,10 +1118,11 @@ async fn get_script_by_path_w_draft(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<ScriptWDraft> {
let path = path.to_path();
check_scopes(&authed, || format!("scripts:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let script_o = sqlx::query_as::<_, ScriptWDraft>(
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email FROM script LEFT JOIN draft ON
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets FROM script LEFT JOIN draft ON
script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
WHERE script.path = $1 AND script.workspace_id = $2
ORDER BY script.created_at DESC LIMIT 1",
@@ -1142,6 +1142,8 @@ async fn get_script_history(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Vec<ScriptHistory>> {
let path = path.to_path();
check_scopes(&authed, || format!("scripts:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let query_result = sqlx::query!(
"SELECT s.hash as hash, dm.deployment_msg as deployment_msg
@@ -1149,7 +1151,7 @@ async fn get_script_history(
WHERE s.workspace_id = $1 AND s.path = $2
ORDER by s.created_at DESC",
w_id,
path.to_path(),
path,
)
.fetch_all(&mut *tx)
.await?;
@@ -1170,6 +1172,8 @@ async fn get_latest_version(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Option<ScriptHistory>> {
let path = path.to_path();
check_scopes(&authed, || format!("scripts:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let row_o = sqlx::query!(
"SELECT s.hash as hash, dm.deployment_msg as deployment_msg
@@ -1177,7 +1181,7 @@ async fn get_latest_version(
WHERE s.workspace_id = $1 AND s.path = $2
ORDER by s.created_at DESC LIMIT 1",
w_id,
path.to_path(),
path,
)
.fetch_optional(&mut *tx)
.await?;
@@ -1200,11 +1204,14 @@ async fn update_script_history(
Path((w_id, script_hash, script_path)): Path<(String, ScriptHash, StripPath)>,
Json(script_history_update): Json<ScriptHistoryUpdate>,
) -> Result<()> {
let script_path = script_path.to_path();
check_scopes(&authed, || format!("scripts:write:{}", script_path))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, script_hash, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET deployment_msg = $4",
w_id,
script_path.to_path(),
script_path,
script_hash.0,
script_history_update.deployment_msg,
)
@@ -1353,6 +1360,7 @@ async fn raw_script_by_path_internal(
unpin: bool,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("scripts:read:{}", path))?;
if !path.ends_with(".py")
&& !path.ends_with(".ts")
&& !path.ends_with(".go")
@@ -1502,6 +1510,9 @@ async fn get_script_by_hash(
}),
)
.await?;
check_scopes(&authed, || format!("scripts:read:{}", &r.script.path))?;
tx.commit().await?;
Ok(Json(r))
@@ -1566,6 +1577,7 @@ async fn archive_script_by_path(
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<()> {
let path = path.to_path();
check_scopes(&authed, || format!("scripts:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
require_owner_of_path(&authed, path)?;
@@ -1638,6 +1650,7 @@ async fn archive_script_by_hash(
.await
.map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?;
check_scopes(&authed, || format!("scripts:write:{}", &script.path))?;
sqlx::query!(
"DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)",
&w_id,
@@ -1686,6 +1699,7 @@ async fn delete_script_by_hash(
.await
.map_err(|e| Error::internal_err(format!("deleting script by hash {w_id}: {e:#}")))?;
check_scopes(&authed, || format!("scripts:write:{}", &script.path))?;
sqlx::query!(
"DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)",
&w_id,
@@ -1728,6 +1742,7 @@ async fn delete_script_by_path(
Query(query): Query<DeleteScriptQuery>,
) -> JsonResult<String> {
let path = path.to_path();
check_scopes(&authed, || format!("scripts:write:{}", path))?;
if path == "u/admin/hub_sync" && w_id == "admins" {
return Err(Error::BadRequest(
backend/windmill-api/src/token.rs
+195
View File
@@ -0,0 +1,195 @@
use axum::{routing::get, Json, Router};
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use windmill_common::error::JsonResult;
#[derive(Default, Serialize, Deserialize, Clone)]
pub struct ScopeOption {
pub value: String,
pub label: String,
pub requires_resource_path: bool,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct ScopeDomain {
pub name: String,
pub description: Option<String>,
pub scopes: Vec<ScopeOption>,
}
fn build_trigger_scope_domains() -> Vec<ScopeDomain> {
const TRIGGER_DOMAINS: &[(&str, &str)] = &[
("http_triggers", "HTTP"),
("websocket_triggers", "WebSocket"),
("kafka_triggers", "Kafka"),
("nats_triggers", "NATS"),
("mqtt_triggers", "MQTT"),
("sqs_triggers", "AWS SQS"),
("gcp_triggers", "GCP Pub/Sub"),
("postgres_triggers", "PostgreSQL"),
];
TRIGGER_DOMAINS
.iter()
.map(|(domain, display_name)| ScopeDomain {
name: format!("{} Triggers", display_name),
description: Some(format!("{} trigger management", display_name)),
scopes: vec![
ScopeOption {
value: format!("{domain}:read"),
label: "Read".to_string(),
requires_resource_path: true,
},
ScopeOption {
value: format!("{domain}:write"),
label: "Write".to_string(),
requires_resource_path: true,
},
],
})
.collect()
}
fn build_standard_scope_domains() -> Vec<ScopeDomain> {
const STANDARD_DOMAINS: &[(&str, &str, &str, bool)] = &[
(
"scripts",
"Scripts",
"Access to automation scripts and workflows",
true,
),
(
"flows",
"Flows",
"Access to automation scripts and workflows",
true,
),
("apps", "Apps", "App management", true),
("raw_apps", "RawApps", "Raw app management", true),
("resources", "Resources", "Resource management", true),
("variables", "Variables", "", true),
(
"schedules",
"Schedules",
"Scheduled tasks and automated triggers",
true,
),
("folders", "Folders", "Folder management", true),
("users", "Users", "User account management", false),
("groups", "Groups", "Group management", false),
("workspaces", "Workspaces", "Workspace management", false),
("audit", "Audit", "Audit log management", false),
("workers", "Workers", "Worker management", false),
("settings", "Settings", "System settings management", false),
(
"service_logs",
"Service Logs",
"Service log management",
false,
),
("configs", "Configs", "Configuration management", false),
("oauth", "OAuth", "OAuth management", false),
("ai", "AI", "AI feature management", false),
(
"agent_workers",
"Agent Workers",
"Agent worker management",
false,
),
("drafts", "Drafts", "Draft management", false),
("favorites", "Favorites", "Favorite items management", false),
("inputs", "Inputs", "Input management", false),
("job_helpers", "Job Helpers", "Job helper utilities", false),
(
"openapi",
"OpenAPI",
"OpenAPI documentation management",
false,
),
("capture", "Capture", "Request capture management", false),
(
"concurrency_groups",
"Concurrency Groups",
"Concurrency group management",
false,
),
("oidc", "OIDC", "OIDC management", false),
("acls", "ACLs", "Access Control List management", false),
("indexer", "Indexer", "Search indexer management", false),
("teams", "Teams", "Team management", false),
(
"git_sync",
"Git Sync",
"Git synchronization management",
false,
),
];
STANDARD_DOMAINS
.iter()
.map(|(key, name, desc, req)| ScopeDomain {
name: name.to_string(),
description: if desc.is_empty() {
None
} else {
Some(desc.to_string())
},
scopes: vec![
ScopeOption {
value: format!("{key}:read"),
label: "Read".to_string(),
requires_resource_path: *req,
},
ScopeOption {
value: format!("{key}:write"),
label: "Write".to_string(),
requires_resource_path: *req,
},
],
})
.collect()
}
lazy_static! {
static ref ALL_SCOPES: Vec<ScopeDomain> = {
let mut groups = vec![ScopeDomain {
name: "Jobs".to_string(),
description: Some("Job management".to_string()),
scopes: vec![
ScopeOption {
value: "jobs:read".to_string(),
label: "Read".to_string(),
requires_resource_path: false,
},
ScopeOption {
value: "jobs:write".to_string(),
label: "Write".to_string(),
requires_resource_path: false,
},
ScopeOption {
value: "jobs:run:scripts".to_string(),
label: "Run scripts".to_string(),
requires_resource_path: true,
},
ScopeOption {
value: "jobs:run:flows".to_string(),
label: "Run flows".to_string(),
requires_resource_path: true,
},
],
}];
groups.extend(build_standard_scope_domains());
groups.extend(build_trigger_scope_domains());
groups
};
}
pub fn global_service() -> Router {
Router::new().route("/list/scopes", get(get_all_available_scopes))
}
async fn get_all_available_scopes() -> JsonResult<Vec<ScopeDomain>> {
Ok(Json(ALL_SCOPES.clone()))
}
backend/windmill-api/src/users.rs
-17
View File
@@ -153,23 +153,6 @@ pub async fn maybe_refresh_folders(
}
}
pub fn check_scopes<F>(authed: &ApiAuthed, required: F) -> error::Result<()>
where
F: FnOnce() -> String,
{
if authed.scopes.as_ref().is_some_and(|scopes| {
scopes
.iter()
.any(|s| s.starts_with("jobs:") || s.starts_with("run:"))
}) {
let req = &required();
if !authed.scopes.as_ref().unwrap().contains(req) {
return Err(Error::BadRequest(format!("missing required scope: {req}")));
}
}
Ok(())
}
pub fn get_scope_tags(authed: &ApiAuthed) -> Option<Vec<&str>> {
authed.scopes.as_ref()?.iter().find_map(|s| {
if s.starts_with("if_jobs:filter_tags:") {
backend/windmill-api/src/utils.rs
+33 -3
View File
@@ -8,7 +8,7 @@
use axum::{body::Body, response::Response};
use regex::Regex;
use serde::{Deserialize, Deserializer};
use serde::{Deserialize, Deserializer, Serialize};
use sqlx::{Postgres, Transaction};
#[cfg(feature = "enterprise")]
use windmill_common::worker::CLOUD_HOSTED;
@@ -18,6 +18,8 @@ use windmill_common::{
DB,
};
use crate::{db::ApiAuthed, scopes::ScopeDefinition};
#[cfg(feature = "enterprise")]
use windmill_common::error::JsonResult;
@@ -41,6 +43,36 @@ pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> {
}
}
pub fn check_scopes<F>(authed: &ApiAuthed, required: F) -> error::Result<()>
where
F: FnOnce() -> String,
{
if let Some(scopes) = authed.scopes.as_ref() {
let mut is_scoped_token = false;
let required_scope = ScopeDefinition::from_scope_string(&required())?;
for scope in scopes {
if !scope.starts_with("if_jobs:filter_tags:") {
if !is_scoped_token {
is_scoped_token = true;
}
match ScopeDefinition::from_scope_string(scope) {
Ok(scope) if scope.includes(&required_scope) => return Ok(()),
_ => {}
}
}
}
if is_scoped_token {
return Err(Error::NotAuthorized(format!(
"Required scope: {}",
required_scope.as_string()
)));
}
}
Ok(())
}
pub async fn require_devops_role(db: &DB, email: &str) -> error::Result<()> {
let is_devops = is_devops_email(db, email).await?;
@@ -192,8 +224,6 @@ where
Ok(o.filter(|s| !s.trim().is_empty()))
}
use serde::Serialize;
#[derive(Serialize)]
pub struct CriticalAlert {
id: i32,
backend/windmill-api/src/variables.rs
+9 -1
View File
@@ -7,7 +7,10 @@
*/
use crate::{
db::{ApiAuthed, DB}, users::{maybe_refresh_folders, require_owner_of_path}, webhook_util::{WebhookMessage, WebhookShared}
db::{ApiAuthed, DB},
users::{maybe_refresh_folders, require_owner_of_path},
utils::check_scopes,
webhook_util::{WebhookMessage, WebhookShared},
};
use axum::{
@@ -135,6 +138,7 @@ async fn get_variable(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<ListableVariable> {
let path = path.to_path();
check_scopes(&authed, || format!("variables:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let variable_o = sqlx::query_as::<_, ListableVariable>(
@@ -219,6 +223,7 @@ async fn get_value(
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<String> {
let path = path.to_path();
check_scopes(&authed, || format!("variables:read:{}", path))?;
let tx = user_db.begin(&authed).await?;
return get_value_internal(tx, &db, &w_id, &path, &authed)
.await
@@ -309,6 +314,7 @@ async fn create_variable(
Query(AlreadyEncrypted { already_encrypted }): Query<AlreadyEncrypted>,
Json(variable): Json<CreateVariable>,
) -> Result<(StatusCode, String)> {
check_scopes(&authed, || format!("variables:write:{}", variable.path))?;
if *CLOUD_HOSTED {
let nb_variables = sqlx::query_scalar!(
"SELECT COUNT(*) FROM variable WHERE workspace_id = $1",
@@ -405,6 +411,7 @@ async fn delete_variable(
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("variables:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
@@ -479,6 +486,7 @@ async fn update_variable(
use sql_builder::prelude::*;
let path = path.to_path();
check_scopes(&authed, || format!("variables:write:{}", path))?;
let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await;
let mut sqlb = SqlBuilder::update_table("variable");
backend/windmill-api/src/websocket_triggers.rs
+10 -2
View File
@@ -36,6 +36,7 @@ use crate::{
db::{ApiAuthed, DB},
trigger_helpers::{trigger_runnable, trigger_runnable_and_wait_for_raw_result, TriggerJobArgs},
users::fetch_api_authed,
utils::check_scopes,
};
use std::borrow::Cow;
@@ -183,8 +184,9 @@ async fn get_websocket_trigger(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> error::JsonResult<WebsocketTrigger> {
let mut tx = user_db.begin(&authed).await?;
let path = path.to_path();
check_scopes(&authed, || format!("websocket_triggers:read:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let trigger = sqlx::query_as::<_, WebsocketTrigger>(
r#"SELECT *
FROM websocket_trigger
@@ -208,6 +210,8 @@ async fn create_websocket_trigger(
Path(w_id): Path<String>,
Json(ct): Json<NewWebsocketTrigger>,
) -> error::Result<(StatusCode, String)> {
check_scopes(&authed, || format!("websocket_triggers:write:{}", ct.path))?;
if *CLOUD_HOSTED {
return Err(error::Error::BadRequest(
"WebSocket triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(),
@@ -300,6 +304,7 @@ async fn update_websocket_trigger(
Json(ct): Json<EditWebsocketTrigger>,
) -> error::Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("websocket_triggers:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
let filters = ct.filters.into_iter().map(SqlxJson).collect_vec();
@@ -371,8 +376,9 @@ pub async fn set_enabled(
Path((w_id, path)): Path<(String, StripPath)>,
Json(payload): Json<SetEnabled>,
) -> error::Result<String> {
let mut tx = user_db.begin(&authed).await?;
let path = path.to_path();
check_scopes(&authed, || format!("websocket_triggers:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
// important to set server_id, last_server_ping and error to NULL to stop current websocket listener
let one_o = sqlx::query_scalar!(
@@ -424,6 +430,8 @@ async fn delete_websocket_trigger(
Path((w_id, path)): Path<(String, StripPath)>,
) -> error::Result<String> {
let path = path.to_path();
check_scopes(&authed, || format!("websocket_triggers:write:{}", path))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"DELETE FROM websocket_trigger WHERE workspace_id = $1 AND path = $2",
backend/windmill-common/Cargo.toml
+3
View File
@@ -102,3 +102,6 @@ tonic = { workspace = true, optional = true }
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemalloc-ctl = { optional = true, workspace = true }
[target.'cfg(windows)'.dependencies]
sysinfo.workspace = true
backend/windmill-common/src/assets.rs
+11 -68
View File
@@ -1,8 +1,7 @@
use serde::{Deserialize, Serialize};
use sqlx::PgExecutor;
use windmill_parser::asset_parser::ParseAssetsResult;
use crate::{error, scripts::ScriptLang};
use crate::error;
#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)]
#[sqlx(type_name = "ASSET_KIND", rename_all = "lowercase")]
@@ -10,7 +9,8 @@ use crate::{error, scripts::ScriptLang};
pub enum AssetKind {
S3Object,
Resource,
Variable,
// Avoid unnexpected crashes when deserializing old assets
Variable, // Deprecated
}
#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)]
@@ -41,85 +41,28 @@ pub struct AssetUsage {
pub access_type: AssetUsageAccessType,
}
#[derive(Serialize, Deserialize, Debug, Clone, Hash)]
pub struct AssetWithAccessType {
#[derive(Serialize, Deserialize, Debug, Clone, Hash, sqlx::Type)]
pub struct AssetWithAltAccessType {
pub path: String,
pub kind: AssetKind,
pub access_type: AssetUsageAccessType,
}
pub fn parse_assets(
input: &str,
lang: ScriptLang,
) -> anyhow::Result<Option<Vec<ParseAssetsResult<String>>>> {
let r = match lang {
ScriptLang::Python3 => windmill_parser_py::parse_assets(input),
ScriptLang::DuckDb => windmill_parser_sql::parse_assets(input).map(|a| {
a.iter()
.map(|a| ParseAssetsResult {
path: a.path.to_string(),
access_type: a.access_type,
kind: a.kind,
})
.collect()
}),
ScriptLang::Deno | ScriptLang::Bun | ScriptLang::Nativets => {
windmill_parser_ts::parse_assets(input)
}
_ => return Ok(None),
};
return r.map(Some);
}
impl From<windmill_parser::asset_parser::AssetKind> for AssetKind {
fn from(kind: windmill_parser::asset_parser::AssetKind) -> Self {
match kind {
windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object,
windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource,
windmill_parser::asset_parser::AssetKind::Variable => AssetKind::Variable,
}
}
}
impl From<windmill_parser::asset_parser::AssetUsageAccessType> for AssetUsageAccessType {
fn from(access_type: windmill_parser::asset_parser::AssetUsageAccessType) -> Self {
match access_type {
windmill_parser::asset_parser::AssetUsageAccessType::R => AssetUsageAccessType::R,
windmill_parser::asset_parser::AssetUsageAccessType::W => AssetUsageAccessType::W,
windmill_parser::asset_parser::AssetUsageAccessType::RW => AssetUsageAccessType::RW,
}
}
pub access_type: Option<AssetUsageAccessType>,
pub alt_access_type: Option<AssetUsageAccessType>,
}
pub async fn insert_asset_usage<'e>(
executor: impl PgExecutor<'e>,
workspace_id: &str,
parsed_asset: &ParseAssetsResult<String>,
fallback_access_types: Option<&[AssetWithAccessType]>,
asset: &AssetWithAltAccessType,
usage_path: &str,
usage_kind: AssetUsageKind,
) -> error::Result<()> {
let kind: AssetKind = parsed_asset.kind.into();
let asset_alternative_access_type = || {
fallback_access_types
.as_ref()
.and_then(|v| {
v.iter()
.find(|a| a.kind == kind && a.path == parsed_asset.path)
})
.map(|a| a.access_type)
};
let access_type: Option<AssetUsageAccessType> = parsed_asset
.access_type
.map(Into::into)
.or_else(asset_alternative_access_type);
sqlx::query!(
r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)
VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING"#,
workspace_id,
parsed_asset.path,
kind as AssetKind,
access_type as Option<AssetUsageAccessType>,
asset.path,
asset.kind as AssetKind,
(asset.access_type.or(asset.alt_access_type)) as Option<AssetUsageAccessType>,
usage_path,
usage_kind as AssetUsageKind
)
backend/windmill-common/src/flows.rs
+9 -5
View File
@@ -18,7 +18,7 @@ use sqlx::types::Json;
use sqlx::types::JsonRawValue;
use crate::{
assets::AssetWithAccessType,
assets::AssetWithAltAccessType,
cache,
error::Error,
more_serde::{default_empty_string, default_id, default_null, default_true, is_default},
@@ -506,7 +506,7 @@ pub enum FlowModuleValue {
#[serde(skip_serializing_if = "Option::is_none")]
is_trigger: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
asset_fallback_access_types: Option<Vec<AssetWithAccessType>>,
assets: Option<Vec<AssetWithAltAccessType>>,
},
Identity,
// Internal only, never exposed to the frontend.
@@ -526,6 +526,8 @@ pub enum FlowModuleValue {
concurrency_time_window_s: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
is_trigger: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
assets: Option<Vec<AssetWithAltAccessType>>,
},
}
@@ -560,7 +562,7 @@ struct UntaggedFlowModuleValue {
id: Option<FlowNodeId>,
default_node: Option<FlowNodeId>,
modules_node: Option<FlowNodeId>,
asset_fallback_access_types: Option<Vec<AssetWithAccessType>>,
assets: Option<Vec<AssetWithAltAccessType>>,
}
impl<'de> Deserialize<'de> for FlowModuleValue {
@@ -635,7 +637,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
concurrent_limit: untagged.concurrent_limit,
concurrency_time_window_s: untagged.concurrency_time_window_s,
is_trigger: untagged.is_trigger,
asset_fallback_access_types: untagged.asset_fallback_access_types,
assets: untagged.assets,
}),
"flowscript" => Ok(FlowModuleValue::FlowScript {
input_transforms: untagged.input_transforms.unwrap_or_default(),
@@ -650,6 +652,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
concurrent_limit: untagged.concurrent_limit,
concurrency_time_window_s: untagged.concurrency_time_window_s,
is_trigger: untagged.is_trigger,
assets: untagged.assets,
}),
"identity" => Ok(FlowModuleValue::Identity),
other => Err(serde::de::Error::unknown_variant(
@@ -785,6 +788,7 @@ pub async fn resolve_module(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
assets,
} = std::mem::replace(&mut val, Identity)
else {
unreachable!()
@@ -808,7 +812,7 @@ pub async fn resolve_module(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
asset_fallback_access_types: None,
assets,
};
}
ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => {
backend/windmill-common/src/lib.rs
+22 -5
View File
@@ -339,7 +339,7 @@ pub async fn connect(
worker_mode: bool,
) -> Result<sqlx::Pool<sqlx::Postgres>, error::Error> {
use std::time::Duration;
use sqlx::Executor;
sqlx::postgres::PgPoolOptions::new()
.min_connections((max_connections / 5).clamp(3, max_connections))
.max_connections(max_connections)
@@ -347,13 +347,30 @@ pub async fn connect(
.after_connect(move |conn, _| {
if worker_mode {
Box::pin(async move {
sqlx::query("SET enable_seqscan = OFF;")
.execute(conn)
.await?;
if let Err(e) = conn.execute(r#"
SET enable_seqscan = OFF;
SET statement_timeout = '5min';
SET idle_in_transaction_session_timeout = '10min';
SET tcp_keepalives_idle = 300;
SET tcp_keepalives_interval = 60;
SET tcp_keepalives_count = 10;"#)
.await {
tracing::error!("Error setting postgres settings: {}", e);
}
Ok(())
})
} else {
Box::pin(async move { Ok(()) })
Box::pin(async move {
if let Err(e) = conn.execute(r#"
SET statement_timeout = '5min';
SET idle_in_transaction_session_timeout = '10min';
SET tcp_keepalives_idle = 300;
SET tcp_keepalives_interval = 60;
SET tcp_keepalives_count = 10;"#)
.await {
tracing::error!("Error setting postgres settings: {}", e);
}
Ok(()) })
}
})
.connect_with(
backend/windmill-common/src/scripts.rs
+6 -2
View File
@@ -13,7 +13,7 @@ use std::{
};
use crate::{
assets::AssetWithAccessType,
assets::AssetWithAltAccessType,
error::{to_anyhow, Error},
utils::http_get_from_hub,
DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL,
@@ -250,6 +250,9 @@ pub struct Script {
pub has_preprocessor: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub on_behalf_of_email: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[sqlx(json(nullable))]
pub assets: Option<Vec<AssetWithAltAccessType>>,
}
#[derive(Serialize, sqlx::FromRow)]
@@ -349,7 +352,8 @@ pub struct NewScript {
pub codebase: Option<String>,
pub has_preprocessor: Option<bool>,
pub on_behalf_of_email: Option<String>,
pub fallback_access_types: Option<Vec<AssetWithAccessType>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub assets: Option<Vec<AssetWithAltAccessType>>,
}
fn lock_deserialize<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
backend/windmill-common/src/worker.rs
+28
View File
@@ -19,6 +19,8 @@ use std::{
str::FromStr,
sync::{atomic::AtomicBool, Arc},
};
#[cfg(windows)]
use sysinfo::System;
use tokio::sync::RwLock;
use uuid::Uuid;
use windmill_macros::annotations;
@@ -759,6 +761,7 @@ fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<
Ok(())
}
#[cfg(not(windows))]
fn get_cgroupv2_path() -> Option<String> {
let cgroup_path: String = parse_file("/proc/self/cgroup")?;
@@ -767,6 +770,7 @@ fn get_cgroupv2_path() -> Option<String> {
.map(|x| format!("/sys/fs/cgroup{}", x.get(1).unwrap().as_str()))
}
#[cfg(not(windows))]
pub fn get_vcpus() -> Option<i64> {
if Path::new("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").exists() {
// cgroup v1
@@ -792,6 +796,14 @@ pub fn get_vcpus() -> Option<i64> {
}
}
#[cfg(windows)]
pub fn get_vcpus() -> Option<i64> {
let mut sys = System::new();
sys.refresh_cpu_all();
(sys.cpus().len() * 100000).try_into().ok()
}
#[cfg(not(windows))]
fn get_memory_from_meminfo() -> Option<i64> {
let memory_info = parse_file::<String>("/proc/meminfo")?;
if memory_info.contains("MemTotal") {
@@ -808,6 +820,7 @@ fn get_memory_from_meminfo() -> Option<i64> {
None
}
#[cfg(not(windows))]
pub fn get_memory() -> Option<i64> {
let memory_limit: Option<i64> =
if Path::new("/sys/fs/cgroup/memory/memory.limit_in_bytes").exists() {
@@ -832,6 +845,14 @@ pub fn get_memory() -> Option<i64> {
}
}
#[cfg(windows)]
pub fn get_memory() -> Option<i64> {
let mut sys = System::new();
sys.refresh_memory();
Some(sys.total_memory() as i64)
}
#[cfg(not(windows))]
pub fn get_worker_memory_usage() -> Option<i64> {
if Path::new("/sys/fs/cgroup/memory/memory.usage_in_bytes").exists() {
// cgroup v1
@@ -869,6 +890,13 @@ pub fn get_worker_memory_usage() -> Option<i64> {
}
}
#[cfg(windows)]
pub fn get_worker_memory_usage() -> Option<i64> {
let mut sys = System::new();
sys.refresh_memory();
Some(sys.used_memory() as i64)
}
pub fn get_windmill_memory_usage() -> Option<i64> {
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
{
backend/windmill-queue/src/jobs.rs
+3 -2
View File
@@ -3691,7 +3691,8 @@ pub async fn push<'c, 'd>(
None,
None,
),
JobPayload::RawFlowDependencies { path, flow_value } => (
JobPayload::RawFlowDependencies { path, flow_value } => {
(
None,
Some(path),
None,
@@ -3705,7 +3706,7 @@ pub async fn push<'c, 'd>(
None,
None,
None,
),
)},
JobPayload::FlowDependencies { path, dedicated_worker, version } => {
// Keep inserting `value` if not all workers are updated.
// Starting at `v1.440`, the value is fetched on pull from the version id.
backend/windmill-worker/loader_builder.bun.js
+62 -64
View File
@@ -1,9 +1,15 @@
const bo = await Bun.build({
entrypoints: ["./main.ts"],
outdir: "./out",
plugins: [p],
external: ["*"],
});
try {
await Bun.build({
entrypoints: ["./main.ts"],
outdir: "./out",
plugins: [p],
external: ["*"],
});
} catch (err) {
console.log(err);
console.log("Failed to build bundle");
process.exit(1);
}
const fs = require("fs/promises");
@@ -12,66 +18,58 @@ const captureVersion =
import { semver } from "bun";
if (!bo.success) {
bo.logs.forEach((l) => console.log(l));
console.log("Failed to build bundle");
process.exit(1);
} else {
let content = await fs.readFile("./out/main.js", { encoding: "utf8" });
const imports = new Bun.Transpiler().scanImports(
content.replaceAll("__require", "require")
);
let content = await fs.readFile("./out/main.js", { encoding: "utf8" });
const imports = new Bun.Transpiler().scanImports(
content.replaceAll("__require", "require")
);
const dependencies = {};
for (const i of imports) {
let [_, name, version] = i.path.match(captureVersion) ?? [];
if (name == undefined) {
throw Error("Unrecognized import: " + i.path);
}
if (name.startsWith("node:")) {
continue;
}
let splitted = name.split("/");
if (splitted.length > 2) {
name = splitted.slice(0, 2).join("/");
}
if (version == undefined) {
if (dependencies[name] == undefined) {
dependencies[name] = [];
}
} else {
if (dependencies[name] == undefined) {
dependencies[name] = [version];
} else if (!dependencies[name].includes(version)) {
dependencies[name].push(version);
}
}
const dependencies = {};
for (const i of imports) {
let [_, name, version] = i.path.match(captureVersion) ?? [];
if (name == undefined) {
throw Error("Unrecognized import: " + i.path);
}
const resolvedDeps = {};
for (const i in dependencies) {
const versions = dependencies[i];
resolvedDeps[i] =
versions.length == 0
? "latest"
: versions.length == 1
? versions[0]
: reduceIntersect(versions, i);
if (name.startsWith("node:")) {
continue;
}
await Bun.write(
"./package.json",
JSON.stringify({ dependencies: resolvedDeps }, null, 2)
);
function reduceIntersect(versions, name) {
console.log(
`multiple versions detected for ${name}: ${versions.join(", ")}`
);
const regex = /^(?:\^|~|<|<=|>=|>)+/g;
const r = versions
.map((x) => [x, x.replace(regex, "")])
.sort((a, b) => semver.order(a[1], b[1]))[0][0];
console.log(`resolved to ${r} for ${name}`);
return r;
let splitted = name.split("/");
if (splitted.length > 2) {
name = splitted.slice(0, 2).join("/");
}
if (version == undefined) {
if (dependencies[name] == undefined) {
dependencies[name] = [];
}
} else {
if (dependencies[name] == undefined) {
dependencies[name] = [version];
} else if (!dependencies[name].includes(version)) {
dependencies[name].push(version);
}
}
}
const resolvedDeps = {};
for (const i in dependencies) {
const versions = dependencies[i];
resolvedDeps[i] =
versions.length == 0
? "latest"
: versions.length == 1
? versions[0]
: reduceIntersect(versions, i);
}
await Bun.write(
"./package.json",
JSON.stringify({ dependencies: resolvedDeps }, null, 2)
);
function reduceIntersect(versions, name) {
console.log(`multiple versions detected for ${name}: ${versions.join(", ")}`);
const regex = /^(?:\^|~|<|<=|>=|>)+/g;
const r = versions
.map((x) => [x, x.replace(regex, "")])
.sort((a, b) => semver.order(a[1], b[1]))[0][0];
console.log(`resolved to ${r} for ${name}`);
return r;
}
backend/windmill-worker/src/bun_executor.rs
+26 -26
View File
@@ -418,17 +418,17 @@ try {{
}}
const bo = await Bun.build({{
entrypoints: ["{job_dir}/wrapper.mjs"],
outdir: "./",
target: "node",
plugins: [p],
external: fileNames,
minify: true,
}});
if (!bo.success) {{
bo.logs.forEach((l) => console.log(l));
try {{
await Bun.build({{
entrypoints: ["{job_dir}/wrapper.mjs"],
outdir: "./",
target: "node",
plugins: [p],
external: fileNames,
minify: true,
}});
}} catch(err) {{
console.log(err);
console.log("Failed to build node bundle");
process.exit(1);
}}
@@ -462,21 +462,21 @@ plugin(p)
r#"
{}
const bo = await Bun.build({{
entrypoints: ["{job_dir}/main.ts"],
outdir: "./",
target: "{}",
plugins: [p],
external: ["electron"],
minify: {{
identifiers: false,
syntax: true,
whitespace: false
}},
}});
if (!bo.success) {{
bo.logs.forEach((l) => console.log(l));
try {{
await Bun.build({{
entrypoints: ["{job_dir}/main.ts"],
outdir: "./",
target: "{}",
plugins: [p],
external: ["electron"],
minify: {{
identifiers: false,
syntax: true,
whitespace: false
}},
}});
}} catch(err) {{
console.log(err)
console.log("Failed to build node bundle");
process.exit(1);
}}
backend/windmill-worker/src/worker.rs
+35 -22
View File
@@ -493,7 +493,10 @@ impl JobCompletedSender {
} else {
unbounded_sender
}
.send_async(SendResult { result: SendResultPayload::JobCompleted(jc), time: Instant::now() })
.send_async(SendResult {
result: SendResultPayload::JobCompleted(jc),
time: Instant::now(),
})
.await
.map_err(|_e| {
anyhow::anyhow!("Failed to send job completed to background processor")
@@ -520,9 +523,13 @@ impl JobCompletedSender {
match self {
Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, .. }) => {
if wait_for_capacity {
sender.send_async(SendResult { result: send_result, time: Instant::now() }).await
sender
.send_async(SendResult { result: send_result, time: Instant::now() })
.await
} else {
unbounded_sender.send_async(SendResult { result: send_result, time: Instant::now() }).await
unbounded_sender
.send_async(SendResult { result: send_result, time: Instant::now() })
.await
}
}
Self::Http(_) => {
@@ -855,13 +862,9 @@ pub fn start_interactive_worker_shell(
if now.duration_since(last).as_secs()
> TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION =>
{
Duration::from_secs(
WORKER_SHELL_NAP_TIME_DURATION,
)
}
_ => {
Duration::from_millis(*SLEEP_QUEUE * 10)
Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION)
}
_ => Duration::from_millis(*SLEEP_QUEUE * 10),
};
tokio::select! {
_ = tokio::time::sleep(nap_time) => {
@@ -869,7 +872,7 @@ pub fn start_interactive_worker_shell(
_ = killpill_rx.recv() => {
break;
}
}
}
}
Err(err) => {
@@ -1329,11 +1332,15 @@ pub async fn run_worker(
let mut killed_but_draining_same_worker_jobs = false;
let mut killpill_rx2 = killpill_rx.resubscribe();
loop {
let last_processing_duration_secs = last_processing_duration.load(Ordering::SeqCst);
if last_processing_duration_secs > 5 {
let sleep_duration = if last_processing_duration_secs > 10 { 10 } else { 5 };
let sleep_duration = if last_processing_duration_secs > 10 {
10
} else {
5
};
tracing::warn!(worker = %worker_name, hostname = %hostname, "last bg processor processing duration > {sleep_duration}s: {last_processing_duration_secs}s, throttling next job pull by {sleep_duration}s");
last_processing_duration.store(0, Ordering::SeqCst);
tokio::time::sleep(Duration::from_secs(sleep_duration)).await;
@@ -1510,16 +1517,20 @@ pub async fn run_worker(
last_suspend_first = Instant::now();
}
let job = match timeout(Duration::from_secs(10), pull(
&db,
suspend_first,
&worker_name,
None,
#[cfg(feature = "benchmark")]
&mut bench,
let job = match timeout(
Duration::from_secs(10),
pull(
&db,
suspend_first,
&worker_name,
None,
#[cfg(feature = "benchmark")]
&mut bench,
)
.warn_after_seconds(2),
)
.warn_after_seconds(2))
.await {
.await
{
Ok(job) => job,
Err(e) => {
tracing::error!(worker = %worker_name, hostname = %hostname, "pull timed out after 10s, sleeping for 30s: {e:?}");
@@ -1743,7 +1754,9 @@ pub async fn run_worker(
match symlink_dir(&windows_parent, &windows_target).await {
Ok(_) => {
tracing::info!("Successfully created directory symlink on Windows");
tracing::info!(
"Successfully created directory symlink on Windows"
);
}
Err(e) => {
tracing::warn!("Failed to create symlink_dir on Windows (likely needs admin privileges or Developer Mode): {}", e);
backend/windmill-worker/src/worker_lockfiles.rs
+32 -8
View File
@@ -11,9 +11,7 @@ use serde_json::{json, Value};
use sha2::Digest;
use sqlx::types::Json;
use uuid::Uuid;
use windmill_common::assets::{
clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind,
};
use windmill_common::assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind};
use windmill_common::error::Error;
use windmill_common::error::Result;
use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId};
@@ -695,6 +693,16 @@ pub async fn handle_flow_dependency_job(
})
.flatten();
let raw_deps = job
.args
.as_ref()
.map(|x| {
x.get("raw_deps")
.map(|v| serde_json::from_str::<HashMap<String, String>>(v.get()).ok())
.flatten()
})
.flatten();
// `JobKind::FlowDependencies` job store either:
// - A saved flow version `id` in the `script_hash` column.
// - Preview raw flow in the `queue` or `job` table.
@@ -741,6 +749,7 @@ pub async fn handle_flow_dependency_job(
&nodes_to_relock,
occupancy_metrics,
skip_flow_update,
raw_deps,
)
.await?;
if !errors.is_empty() {
@@ -910,6 +919,7 @@ async fn lock_modules<'c>(
locks_to_reload: &Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
skip_flow_update: bool,
raw_deps: Option<HashMap<String, String>>,
// (modules to replace old seq (even unmmodified ones), new transaction, modified ids) )
) -> Result<(
Vec<FlowModule>,
@@ -934,7 +944,7 @@ async fn lock_modules<'c>(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
asset_fallback_access_types,
assets,
} = e.get_value()?
else {
match e.get_value()? {
@@ -963,6 +973,7 @@ async fn lock_modules<'c>(
locks_to_reload,
occupancy_metrics,
skip_flow_update,
raw_deps.clone(),
))
.await?;
e.value = FlowModuleValue::ForloopFlow {
@@ -998,6 +1009,7 @@ async fn lock_modules<'c>(
locks_to_reload,
occupancy_metrics,
skip_flow_update,
raw_deps.clone(),
))
.await?;
nmodified_ids.extend(inner_modified_ids);
@@ -1025,6 +1037,7 @@ async fn lock_modules<'c>(
locks_to_reload,
occupancy_metrics,
skip_flow_update,
raw_deps.clone(),
))
.await?;
e.value = FlowModuleValue::WhileloopFlow {
@@ -1057,6 +1070,7 @@ async fn lock_modules<'c>(
locks_to_reload,
occupancy_metrics,
skip_flow_update,
raw_deps.clone(),
))
.await?;
nmodified_ids.extend(inner_modified_ids);
@@ -1082,6 +1096,7 @@ async fn lock_modules<'c>(
locks_to_reload,
occupancy_metrics,
skip_flow_update,
raw_deps.clone(),
))
.await?;
errors.extend(ninner_errors);
@@ -1122,12 +1137,11 @@ async fn lock_modules<'c>(
continue;
};
for asset in parse_assets(&content, language)?.iter().flatten() {
for asset in assets.iter().flatten() {
insert_asset_usage(
&mut *tx,
&job.workspace_id,
asset,
asset_fallback_access_types.as_ref().map(Vec::as_slice),
job_path,
AssetUsageKind::Flow,
)
@@ -1157,6 +1171,14 @@ async fn lock_modules<'c>(
create_dir_all(job_dir).map_err(|e| {
Error::ExecutionErr(format!("Error creating job dir for flow step lock: {e}"))
})?;
// If we have local lockfiles (and they are enabled) we will replace script content with lockfile and tell hander that it is raw_deps job
let (content, raw_deps) = raw_deps
.as_ref()
.and_then(|llfs| llfs.get(language.as_str()))
.map(|lock| (lock.to_owned(), true))
.unwrap_or((content, false));
let new_lock = capture_dependency_job(
&job.id,
&language,
@@ -1174,7 +1196,7 @@ async fn lock_modules<'c>(
"{}/flow",
&path.clone().unwrap_or_else(|| job_path.to_string())
),
false,
raw_deps,
None,
occupancy_metrics,
)
@@ -1241,7 +1263,7 @@ async fn lock_modules<'c>(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
asset_fallback_access_types,
assets,
});
new_flow_modules.push(e);
@@ -1393,6 +1415,7 @@ async fn reduce_flow<'c>(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
assets,
..
} = std::mem::replace(&mut val, Identity)
else {
@@ -1411,6 +1434,7 @@ async fn reduce_flow<'c>(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
assets,
};
}
ForloopFlow { modules, modules_node, .. }
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.507.1";
export const VERSION = "v1.508.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/apps.ts
+2 -1
View File
@@ -1,5 +1,6 @@
// deno-lint-ignore-file no-explicit-any
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import { requireLogin } from "./auth.ts";
import { resolveWorkspace, validatePath } from "./context.ts";
import { colors, Command, log, SEP, Table, yamlParseFile } from "./deps.ts";
import * as wmill from "./gen/services.gen.ts";
import { ListableApp, Policy } from "./gen/types.gen.ts";
cli/auth.ts
+61
View File
@@ -0,0 +1,61 @@
// deno-lint-ignore-file no-explicit-any
import { colors, log, setClient } from "./deps.ts";
import * as wmill from "./gen/services.gen.ts";
import { GlobalUserInfo } from "./gen/types.gen.ts";
import { loginInteractive, tryGetLoginInfo } from "./login.ts";
import { GlobalOptions } from "./types.ts";
import {
addWorkspace,
removeWorkspace,
Workspace
} from "./workspace.ts";
/**
* Main authentication function - moved from context.ts to break circular dependencies
* This function maintains the original API signature from context.ts
*/
export async function requireLogin(
opts: GlobalOptions
): Promise<GlobalUserInfo> {
// Import resolveWorkspace to avoid circular dependency at module level
const { resolveWorkspace } = await import("./context.ts");
const workspace = await resolveWorkspace(opts);
let token = await tryGetLoginInfo(opts);
if (!token) {
token = workspace.token;
}
setClient(token, workspace.remote.substring(0, workspace.remote.length - 1));
try {
return await wmill.globalWhoami();
} catch (error) {
// Check for network errors and provide clearer messages
const errorMsg = error instanceof Error ? error.message : String(error);
if (errorMsg.includes('fetch') || errorMsg.includes('connection') || errorMsg.includes('ECONNREFUSED') || errorMsg.includes('refused')) {
throw new Error(`Network error: Could not connect to Windmill server at ${workspace.remote}`);
}
log.info(
"! Could not reach API given existing credentials. Attempting to reauth..."
);
const newToken = await loginInteractive(workspace.remote);
if (!newToken) {
throw new Error("Unauthorized: Could not authenticate with the provided credentials");
}
// Update workspace token
removeWorkspace(workspace.name, false, opts);
workspace.token = newToken;
addWorkspace(workspace, opts);
setClient(
newToken,
workspace.remote.substring(0, workspace.remote.length - 1)
);
return await wmill.globalWhoami();
}
}
cli/build.sh
+1 -1
View File
@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Note for mac OS users: you need to install gnu-sed with `brew install gnu-sed` and use `gsed` instead of `sed`.
./gen_wm_client.sh
deno run -A dnt.ts
cli/conf.ts
+20 -6
View File
@@ -108,31 +108,45 @@ export function getEffectiveSettings(
): SyncOptions {
// Start with empty object - no defaults
let effective = {} as SyncOptions;
// Merge top-level settings from config (which contains user's chosen defaults)
Object.keys(config).forEach(key => {
if (key !== 'overrides' && config[key as keyof SyncOptions] !== undefined) {
(effective as any)[key] = config[key as keyof SyncOptions];
}
});
if (!config.overrides) {
if (repo) {
log.info(`No overrides found in wmill.yaml, using top-level settings (repository flag ignored)`);
}
return effective;
}
// Construct override keys using the single format
const workspaceKey = `${baseUrl}:${workspaceId}:*`;
const repoKey = `${baseUrl}:${workspaceId}:${repo}`;
let appliedOverrides: string[] = [];
// Apply workspace-level overrides
if (config.overrides[workspaceKey]) {
Object.assign(effective, config.overrides[workspaceKey]);
appliedOverrides.push("workspace-level");
}
// Apply repository-specific overrides (overrides workspace-level)
if (config.overrides[repoKey]) {
Object.assign(effective, config.overrides[repoKey]);
appliedOverrides.push("repository-specific");
} else if (repo) {
// Repository was specified but no override found
log.info(`Repository override not found for "${repo}", using ${appliedOverrides.length > 0 ? appliedOverrides.join(" + ") : "top-level"} settings`);
}
if (appliedOverrides.length > 0) {
log.info(`Applied ${appliedOverrides.join(" + ")} overrides${repo ? ` for repository "${repo}"` : ""}`);
}
return effective;
}
cli/context.ts
+7 -46
View File
@@ -58,12 +58,12 @@ export async function resolveWorkspace(
if (opts.baseUrl) {
if (opts.workspace && opts.token) {
const normalizedBaseUrl = new URL(opts.baseUrl).toString(); // add trailing slash if not present
// Try to find existing workspace profile by name, then by workspaceId + remote
if (opts.workspace) {
// Try by workspace name first
let existingWorkspace = await getWorkspaceByName(opts.workspace, opts.configDir);
// If not found by name, try to find by workspaceId + remote match
if (!existingWorkspace) {
const { allWorkspaces } = await import("./workspace.ts");
@@ -71,13 +71,13 @@ export async function resolveWorkspace(
const matchingWorkspaces = workspaces.filter(
w => w.workspaceId === opts.workspace && w.remote === normalizedBaseUrl
);
// Due to uniqueness constraint, there can only be 0 or 1 match
if (matchingWorkspaces.length === 1) {
existingWorkspace = matchingWorkspaces[0];
}
}
if (existingWorkspace) {
// Validate that the base URL matches the profile's remote
if (existingWorkspace.remote !== normalizedBaseUrl) {
@@ -95,7 +95,7 @@ export async function resolveWorkspace(
};
}
}
// No existing profile found, create temporary workspace
return {
remote: normalizedBaseUrl,
@@ -121,45 +121,6 @@ export async function resolveWorkspace(
}
}
export async function requireLogin(
opts: GlobalOptions
): Promise<GlobalUserInfo> {
const workspace = await resolveWorkspace(opts);
let token = await tryGetLoginInfo(opts);
if (!token) {
token = workspace.token;
}
setClient(token, workspace.remote.substring(0, workspace.remote.length - 1));
try {
return await wmill.globalWhoami();
} catch (error) {
// Check for network errors and provide clearer messages
const errorMsg = error instanceof Error ? error.message : String(error);
if (errorMsg.includes('fetch') || errorMsg.includes('connection') || errorMsg.includes('ECONNREFUSED') || errorMsg.includes('refused')) {
throw new Error(`Network error: Could not connect to Windmill server at ${workspace.remote}`);
}
log.info(
"! Could not reach API given existing credentials. Attempting to reauth..."
);
const newToken = await loginInteractive(workspace.remote);
if (!newToken) {
throw new Error("Unauthorized: Could not authenticate with the provided credentials");
}
removeWorkspace(workspace.name, false, opts);
workspace.token = newToken;
addWorkspace(workspace, opts);
setClient(
newToken,
workspace.remote.substring(0, workspace.remote.length - 1)
);
return await wmill.globalWhoami();
}
}
export async function fetchVersion(baseUrl: string): Promise<string> {
const requestHeaders = new Headers();
@@ -176,13 +137,13 @@ export async function fetchVersion(baseUrl: string): Promise<string> {
new URL(new URL(baseUrl).origin + "/api/version"),
{ headers: requestHeaders, method: "GET" }
);
if (!response.ok) {
// Consume response body even on error to avoid resource leak
await response.text();
throw new Error(`Failed to fetch version: ${response.status} ${response.statusText}`);
}
return await response.text();
}
export async function tryResolveVersion(
cli/dev.ts
+2 -1
View File
@@ -12,7 +12,8 @@ import {
} from "./deps.ts";
import { getTypeStrFromPath, GlobalOptions } from "./types.ts";
import { ignoreF } from "./sync.ts";
import { requireLogin, resolveWorkspace } from "./context.ts";
import { requireLogin } from "./auth.ts";
import { resolveWorkspace } from "./context.ts";
import {
SyncOptions,
mergeConfigWithConfigFile,
cli/flow.ts
+13 -5
View File
@@ -4,10 +4,11 @@ import { Confirm, SEP, log, yamlStringify } from "./deps.ts";
import { colors, Command, Table, yamlParseFile } from "./deps.ts";
import * as wmill from "./gen/services.gen.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import { requireLogin } from "./auth.ts";
import { resolveWorkspace, validatePath } from "./context.ts";
import { resolve, track_job } from "./script.ts";
import { defaultFlowDefinition } from "./bootstrap/flow_bootstrap.ts";
import { generateFlowLockInternal } from "./metadata.ts";
import { blueColor, generateFlowLockInternal } from "./metadata.ts";
import { SyncOptions, mergeConfigWithConfigFile } from "./conf.ts";
import { FSFSElement, elementsToMap, ignoreF } from "./sync.ts";
import { readInlinePathSync } from "./utils.ts";
@@ -238,15 +239,18 @@ async function run(
async function generateLocks(
opts: GlobalOptions & {
yes?: boolean;
useRawRequirements?: boolean;
} & SyncOptions,
folder: string | undefined
) {
const useRawReqs = opts.useRawRequirements || Deno.env.get("USE_RAW_REQUIREMENTS") === "true";
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
opts = await mergeConfigWithConfigFile(opts);
if (folder) {
// read script metadata file
await generateFlowLockInternal(folder, false, workspace);
await generateFlowLockInternal(folder, false, workspace, opts, undefined, undefined, useRawReqs);
} else {
const ignore = await ignoreF(opts);
const elems = Object.keys(
@@ -267,7 +271,7 @@ async function generateLocks(
let hasAny = false;
for (const folder of elems) {
const candidate = await generateFlowLockInternal(folder, true, workspace);
const candidate = await generateFlowLockInternal(folder, true, workspace, opts, undefined, undefined, useRawReqs);
if (candidate) {
hasAny = true;
log.info(colors.green(`+ ${candidate}`));
@@ -289,7 +293,7 @@ async function generateLocks(
return;
}
for (const folder of elems) {
await generateFlowLockInternal(folder, false, workspace);
await generateFlowLockInternal(folder, false, workspace, opts,undefined, undefined, useRawReqs);
}
}
}
@@ -348,6 +352,10 @@ const command = new Command()
)
.arguments("[flow:file]")
.option("--yes", "Skip confirmation prompt")
.option(
"-r --use-raw-requirements",
"Use raw requirements (requirements.txt, go.mod, package.json, etc) instead of generating them on the server (can also be set with USE_RAW_REQUIREMENTS=true environment variable)"
)
.option(
"-i --includes <patterns:file[]>",
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)"
cli/folder.ts
+2 -1
View File
@@ -2,7 +2,8 @@
import { colors, Command, log, SEP, Table } from "./deps.ts";
import * as wmill from "./gen/services.gen.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import { requireLogin } from "./auth.ts";
import { resolveWorkspace, validatePath } from "./context.ts";
import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts";
import { Folder } from "./gen/types.gen.ts";
cli/gen_wm_client.sh
+1 -1
View File
@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -eou pipefail
script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cli/gitsync-settings.ts
+284 -55
View File
@@ -1,6 +1,7 @@
import { colors, Command, log, yamlStringify } from "./deps.ts";
import { colors, Command, Confirm, log, yamlStringify } from "./deps.ts";
import { GlobalOptions } from "./types.ts";
import { requireLogin, resolveWorkspace } from "./context.ts";
import { requireLogin } from "./auth.ts";
import { resolveWorkspace } from "./context.ts";
import * as wmill from "./gen/services.gen.ts";
import {
DEFAULT_SYNC_OPTIONS,
@@ -163,6 +164,147 @@ function includeTypeToSyncOptions(
};
}
// Shared migration function for legacy repositories
async function handleLegacyRepositoryMigration(
selectedRepo: any,
gitSyncSettings: any,
workspace: any,
opts: { yes?: boolean },
operationName: string = "operation"
): Promise<any> {
if (selectedRepo.settings) {
return selectedRepo; // Already migrated
}
// This repository is in legacy format - handle migration
if (!gitSyncSettings.include_path || !gitSyncSettings.include_type) {
throw new Error(
`Repository "${selectedRepo.git_repo_resource_path}" has legacy format but workspace-level include_path or include_type is missing. This indicates corrupted git-sync settings.`
);
}
const workspaceIncludePath = gitSyncSettings.include_path;
const workspaceIncludeType = gitSyncSettings.include_type;
if (Deno.stdout.isTerminal() && !opts.yes) {
// Interactive mode - show migration prompt
console.log(colors.yellow('\n⚠️ Legacy git-sync settings detected!'));
console.log(`\nRepository "${selectedRepo.git_repo_resource_path}" has legacy settings format.`);
console.log('The new format allows per-repository filter configuration.');
if (operationName === "push") {
console.log('This repository must be migrated before pushing settings.\n');
} else {
console.log('\n');
}
console.log(colors.bold('Current workspace-level settings:'));
console.log(` Include paths: ${workspaceIncludePath.join(', ')}`);
console.log(` Include types: ${workspaceIncludeType.join(', ')}\n`);
// Show what the migration will do
let finalIncludeType = [...workspaceIncludeType];
if (selectedRepo.exclude_types_override && selectedRepo.exclude_types_override.length > 0) {
const originalCount = finalIncludeType.length;
finalIncludeType = finalIncludeType.filter(
type => !selectedRepo.exclude_types_override.includes(type)
);
const excludedCount = originalCount - finalIncludeType.length;
console.log(colors.yellow(`Repository excludes ${excludedCount} types: ${selectedRepo.exclude_types_override.join(', ')}`));
}
console.log(colors.bold('\nAfter migration, repository will have:'));
console.log(` Include paths: ${workspaceIncludePath.join(', ')}`);
console.log(` Include types: ${finalIncludeType.join(', ')}\n`);
const confirm = await Confirm.prompt({
message: operationName === "push"
? 'Do you want to migrate this repository before pushing?'
: 'Do you want to migrate this repository?',
default: true
});
if (!confirm) {
const message = operationName === "push"
? '\n⚠️ Migration skipped. Cannot push to legacy repository.'
: '\n⚠️ Migration skipped. You can migrate later via the UI.';
console.log(colors.yellow(message));
if (operationName === "push") {
return null; // Signal to exit push operation
}
throw new Error('Migration cancelled by user');
}
// Perform the migration
let migratedIncludeType = [...workspaceIncludeType];
if (selectedRepo.exclude_types_override && selectedRepo.exclude_types_override.length > 0) {
migratedIncludeType = migratedIncludeType.filter(
type => !selectedRepo.exclude_types_override.includes(type)
);
}
const migratedRepo = {
...selectedRepo,
settings: {
include_path: [...workspaceIncludePath],
include_type: migratedIncludeType,
exclude_path: [],
extra_include_path: []
}
};
// Remove the old field
delete migratedRepo.exclude_types_override;
// Update the backend with migrated repository
const updatedRepositories = gitSyncSettings.repositories.map((repo: any) => {
if (repo.git_repo_resource_path === selectedRepo.git_repo_resource_path) {
return migratedRepo;
}
return repo;
});
await wmill.editWorkspaceGitSyncConfig({
workspace: workspace.workspaceId,
requestBody: {
git_sync_settings: {
repositories: updatedRepositories,
// Keep workspace-level settings if other repos are still legacy
...(gitSyncSettings.repositories.some((r: any) => r.git_repo_resource_path !== selectedRepo.git_repo_resource_path && !r.settings) && {
include_path: workspaceIncludePath,
include_type: workspaceIncludeType
})
}
}
});
console.log(colors.green('\n✓ Repository migration completed successfully!'));
if (operationName === "push") {
console.log('Now proceeding with push operation...\n');
}
return migratedRepo;
} else {
// Non-interactive mode - show error
console.error(colors.red('\n❌ Legacy git-sync settings detected!'));
console.error(`\nRepository "${selectedRepo.git_repo_resource_path}" has legacy settings format.`);
if (operationName === "push") {
console.error('This repository must be migrated before pushing settings.');
}
console.error('Please choose one of the following options:\n');
console.error('1. Go to the Windmill UI > Workspace Settings > Git Sync');
console.error(' Review and save this repository to migrate to the new format.\n');
console.error('2. Run this command in interactive mode (with TTY) to migrate.');
console.error(` Example: wmill gitsync-settings ${operationName}\n`);
if (operationName === "push") {
console.error('3. Pull settings first to migrate: wmill gitsync-settings pull\n');
} else {
console.error('3. Push local settings to override backend settings:');
console.error(' wmill gitsync-settings push\n');
}
Deno.exit(1);
}
}
// Convert SyncOptions boolean flags to backend include_type array
function syncOptionsToIncludeType(opts: SyncOptions): string[] {
const includeTypes: string[] = [];
@@ -278,6 +420,31 @@ function displayChanges(
}
}
async function selectAndLogRepository(
repositories: GitSyncRepository[],
repository?: string,
): Promise<GitSyncRepository> {
let selectedRepo: GitSyncRepository;
if (repository) {
const found = repositories.find(
(r: GitSyncRepository) =>
r.git_repo_resource_path === repository ||
r.git_repo_resource_path === `$res:${repository}`,
);
if (!found) {
throw new Error(`Repository ${repository} not found`);
}
selectedRepo = found;
const repoPath = selectedRepo.git_repo_resource_path.replace(/^\$res:/, "");
log.info(colors.cyan(`Using repository: ${colors.bold(repoPath)}`));
} else {
selectedRepo = await selectRepository(repositories);
}
return selectedRepo;
}
async function pullGitSyncSettings(
opts: GlobalOptions & {
repository?: string;
@@ -288,6 +455,7 @@ async function pullGitSyncSettings(
replace?: boolean;
override?: boolean;
withBackendSettings?: string;
yes?: boolean;
},
) {
const workspace = await resolveWorkspace(opts);
@@ -400,22 +568,19 @@ async function pullGitSyncSettings(
}
// Find the repository to work with
let selectedRepo: GitSyncRepository;
if (opts.repository) {
const found = settings.git_sync.repositories.find(
(r: GitSyncRepository) =>
r.git_repo_resource_path === opts.repository ||
r.git_repo_resource_path === `$res:${opts.repository}`,
);
if (!found) {
throw new Error(`Repository ${opts.repository} not found`);
}
selectedRepo = found;
} else {
selectedRepo = await selectRepository(
settings.git_sync.repositories,
);
}
let selectedRepo = await selectAndLogRepository(
settings.git_sync.repositories,
opts.repository,
);
// Check if the selected repository needs migration and handle it
selectedRepo = await handleLegacyRepositoryMigration(
selectedRepo,
settings.git_sync,
workspace,
opts,
"pull"
);
// Convert backend settings to SyncOptions format
const backendSyncOptions: SyncOptions = {
@@ -533,7 +698,7 @@ async function pullGitSyncSettings(
);
} else {
if (hasChanges) {
log.info("Changes that would be made:");
log.info("Changes that would be applied locally:");
const changes = generateChanges(normalizedCurrent, normalizedBackend);
if (Object.keys(changes).length === 0) {
@@ -572,7 +737,14 @@ async function pullGitSyncSettings(
);
const hasConflict = !deepEqual(gitSyncBackend, gitSyncCurrent);
if (hasConflict && Deno.stdin.isTerminal()) {
if (hasConflict && !opts.yes && Deno.stdin.isTerminal()) {
// Show the diff first
log.info("Changes that would be applied locally:");
const changes = generateChanges(currentSettings, backendSyncOptions);
if (Object.keys(changes).length > 0) {
displayChanges(changes);
}
// Interactive mode - ask user
const { Select } = await import("./deps.ts");
const choice = await Select.prompt({
@@ -606,6 +778,16 @@ async function pullGitSyncSettings(
repoPath,
);
}
} else if (hasConflict && opts.yes) {
// --yes flag: default to override behavior for conflicts
writeMode = "override";
const repoPath = normalizeRepoPath(selectedRepo.git_repo_resource_path);
overrideKey = constructOverrideKey(
workspace.remote,
workspace.workspaceId,
repoPath,
);
log.info(colors.yellow("Settings conflict detected. Using --override behavior (default for --yes)."));
} else if (hasConflict) {
// Non-interactive mode with conflicts - show message and exit
if (opts.jsonOutput) {
@@ -663,11 +845,19 @@ async function pullGitSyncSettings(
// Apply the settings based on write mode
let updatedConfig: SyncOptions;
// Log which settings mode is being used
const repoPath = selectedRepo.git_repo_resource_path.replace(/^\$res:/, "");
if (writeMode === "override") {
log.info(`Applied repository-specific overrides for repository "${repoPath}"`);
} else {
log.info(`Applied settings for repository "${repoPath}"`);
}
if (writeMode === "replace") {
// Preserve existing local config and update only git-sync fields
updatedConfig = { ...localConfig };
// Remove overrides since we're in replace mode
delete updatedConfig.overrides;
// Clear overrides since we're in replace mode, but keep empty object for consistency
updatedConfig.overrides = {};
// Update with backend git-sync settings
Object.assign(updatedConfig, backendSyncOptions);
} else if (writeMode === "override" && overrideKey) {
@@ -760,6 +950,7 @@ async function pushGitSyncSettings(
diff?: boolean;
jsonOutput?: boolean;
withBackendSettings?: string;
yes?: boolean;
},
) {
const workspace = await resolveWorkspace(opts);
@@ -887,21 +1078,23 @@ async function pushGitSyncSettings(
}
// Find the repository to work with
let selectedRepo: GitSyncRepository;
if (opts.repository) {
const found = settings.git_sync.repositories.find(
(r: GitSyncRepository) =>
r.git_repo_resource_path === opts.repository ||
r.git_repo_resource_path === `$res:${opts.repository}`,
);
if (!found) {
throw new Error(`Repository ${opts.repository} not found`);
}
selectedRepo = found;
} else {
selectedRepo = await selectRepository(
settings.git_sync.repositories,
);
let selectedRepo = await selectAndLogRepository(
settings.git_sync.repositories,
opts.repository,
);
// Check if the selected repository needs migration and handle it
selectedRepo = await handleLegacyRepositoryMigration(
selectedRepo,
settings.git_sync,
workspace,
opts,
"push"
);
// If migration was cancelled, exit
if (selectedRepo === null) {
return;
}
// Get effective settings for this workspace/repo
@@ -921,24 +1114,25 @@ async function pushGitSyncSettings(
extra_include_path: effectiveSettings.extraIncludes || [],
};
// Calculate diff for all modes
const currentBackend = selectedRepo.settings;
// Convert current backend settings to SyncOptions for user-friendly display
const currentSyncOptions: SyncOptions = {
includes: currentBackend.include_path || [],
excludes: currentBackend.exclude_path || [],
extraIncludes: currentBackend.extra_include_path || [],
...includeTypeToSyncOptions(currentBackend.include_type || []),
};
const normalizedCurrent = normalizeSyncOptions(currentSyncOptions);
const normalizedEffective = normalizeSyncOptions(effectiveSettings);
const gitSyncCurrent = extractGitSyncFields(normalizedCurrent);
const gitSyncEffective = extractGitSyncFields(normalizedEffective);
const hasChanges = !deepEqual(gitSyncEffective, gitSyncCurrent);
if (opts.diff) {
// Show what would be pushed
const currentBackend = selectedRepo.settings;
// Convert current backend settings to SyncOptions for user-friendly display
const currentSyncOptions: SyncOptions = {
includes: currentBackend.include_path || [],
excludes: currentBackend.exclude_path || [],
extraIncludes: currentBackend.extra_include_path || [],
...includeTypeToSyncOptions(currentBackend.include_type || []),
};
const normalizedCurrent = normalizeSyncOptions(currentSyncOptions);
const normalizedEffective = normalizeSyncOptions(effectiveSettings);
const gitSyncCurrent = extractGitSyncFields(normalizedCurrent);
const gitSyncEffective = extractGitSyncFields(normalizedEffective);
const hasChanges = !deepEqual(gitSyncEffective, gitSyncCurrent);
// --diff flag: show differences and exit
if (opts.jsonOutput) {
// Generate structured diff using the same normalized objects
const structuredDiff = hasChanges
@@ -957,7 +1151,7 @@ async function pushGitSyncSettings(
);
} else {
if (hasChanges) {
log.info("Changes that would be pushed:");
log.info("Changes that would be pushed to Windmill:");
const changes = generateChanges(
normalizedCurrent,
normalizedEffective,
@@ -975,6 +1169,39 @@ async function pushGitSyncSettings(
return;
}
// Default behavior: show changes and ask for confirmation (unless --yes is passed)
if (hasChanges) {
if (!opts.jsonOutput) {
const changes = generateChanges(
normalizedCurrent,
normalizedEffective,
);
if (Object.keys(changes).length === 0) {
log.info(colors.green("No changes to push"));
return;
} else {
log.info("Changes that would be pushed to Windmill:");
displayChanges(changes);
}
}
// Ask for confirmation unless --yes is passed or not in TTY
if (!opts.yes && Deno.stdin.isTerminal()) {
const confirmed = await Confirm.prompt({
message: `Do you want to apply these changes to the remote?`,
default: true,
});
if (!confirmed) {
return;
}
}
} else {
log.info(colors.green("No changes to push"));
return;
}
if (opts.withBackendSettings) {
// Skip backend update when using simulated settings
if (opts.jsonOutput) {
@@ -1103,6 +1330,7 @@ const command = new Command()
"--with-backend-settings <json:string>",
"Use provided JSON settings instead of querying backend (for testing)",
)
.option("--yes", "Skip interactive prompts and use default behavior")
.action(pullGitSyncSettings as any)
.command("push")
.description(
@@ -1118,6 +1346,7 @@ const command = new Command()
"--with-backend-settings <json:string>",
"Use provided JSON settings instead of querying backend (for testing)",
)
.option("--yes", "Skip interactive prompts and use default behavior")
.action(pushGitSyncSettings as any);
export { pullGitSyncSettings, pushGitSyncSettings };
cli/hub.ts
+2 -1
View File
@@ -2,7 +2,8 @@
import { Command, log } from "./deps.ts";
import * as wmill from "./gen/services.gen.ts";
import { requireLogin, resolveWorkspace } from "./context.ts";
import { requireLogin } from "./auth.ts";
import { resolveWorkspace } from "./context.ts";
import { pushResourceType } from "./resource-type.ts";
import { GlobalOptions } from "./types.ts";
import { deepEqual } from "./utils.ts";
cli/main.ts
+4 -4
View File
@@ -65,7 +65,7 @@ export {
// }
// });
export const VERSION = "1.507.1";
export const VERSION = "1.508.0";
const command = new Command()
.name("wmill")
@@ -143,6 +143,7 @@ const command = new Command()
skipFolders: DEFAULT_SYNC_OPTIONS.skipFolders,
includeSchedules: DEFAULT_SYNC_OPTIONS.includeSchedules,
includeTriggers: DEFAULT_SYNC_OPTIONS.includeTriggers,
overrides: {},
};
await Deno.writeTextFile(
@@ -159,9 +160,8 @@ const command = new Command()
// Check for backend git-sync settings unless --use-default is specified
if (!opts.useDefault) {
try {
const { requireLogin, resolveWorkspace } = await import(
"./context.ts"
);
const { requireLogin } = await import("./auth.ts");
const { resolveWorkspace } = await import("./context.ts");
// Check if user has workspace configured
const { getActiveWorkspace } = await import(
cli/metadata.ts
+185 -129
View File
@@ -14,9 +14,9 @@ import {
} from "./bootstrap/script_bootstrap.ts";
import { Workspace } from "./workspace.ts";
import { SchemaProperty } from "./bootstrap/common.ts";
import { ScriptLanguage } from "./script_common.ts";
import { languagesWithRawReqsSupport, LanguageWithRawReqsSupport, ScriptLanguage } from "./script_common.ts";
import { inferContentTypeFromFilePath } from "./script_common.ts";
import { GlobalDeps, exts } from "./script.ts";
import { GlobalDeps, exts, findGlobalDeps } from "./script.ts";
import {
FSFSElement,
extractInlineScriptsForFlows,
@@ -37,43 +37,16 @@ export class LockfileGenerationError extends Error {
}
}
export async function generateAllMetadata() {}
export async function generateAllMetadata() { }
function findClosestRawReqs(
lang: "bun" | "python3" | "php" | "go" | undefined,
lang: LanguageWithRawReqsSupport | undefined,
remotePath: string,
globalDeps: GlobalDeps
globalDeps: GlobalDeps,
): string | undefined {
let bestCandidate: { k: string; v: string } | undefined = undefined;
if (lang == "bun") {
Object.entries(globalDeps.pkgs).forEach(([k, v]) => {
if (
remotePath.startsWith(k) &&
k.length >= (bestCandidate?.k ?? "").length
) {
bestCandidate = { k, v };
}
});
} else if (lang == "python3") {
Object.entries(globalDeps.reqs).forEach(([k, v]) => {
if (
remotePath.startsWith(k) &&
k.length >= (bestCandidate?.k ?? "").length
) {
bestCandidate = { k, v };
}
});
} else if (lang == "php") {
Object.entries(globalDeps.composers).forEach(([k, v]) => {
if (
remotePath.startsWith(k) &&
k.length >= (bestCandidate?.k ?? "").length
) {
bestCandidate = { k, v };
}
});
} else if (lang == "go") {
Object.entries(globalDeps.goMods).forEach(([k, v]) => {
if (lang) {
Object.entries(globalDeps.get(lang) ?? {}).forEach(([k, v]) => {
if (
remotePath.startsWith(k) &&
k.length >= (bestCandidate?.k ?? "").length
@@ -87,12 +60,25 @@ function findClosestRawReqs(
}
const TOP_HASH = "__flow_hash";
async function generateFlowHash(folder: string) {
async function generateFlowHash(
rawReqs: Record<string, string> | undefined,
folder: string,
defaultTs: "bun" | "deno" | undefined
) {
const elems = await FSFSElement(path.join(Deno.cwd(), folder), [], true);
const hashes: Record<string, string> = {};
for await (const f of elems.getChildren()) {
if (exts.some((e) => f.path.endsWith(e))) {
hashes[f.path] = await generateHash(await f.getContentText());
let reqs: string | undefined;
if (rawReqs) {
// Get language name from path
const lang = inferContentTypeFromFilePath(f.path, defaultTs);
// Get lock for that language
[, reqs] = Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? [];
}
// Embed lock into hash
hashes[f.path] = await generateHash(await f.getContentText() + (reqs ?? ""));
}
}
return { ...hashes, [TOP_HASH]: await generateHash(JSON.stringify(hashes)) };
@@ -101,9 +87,13 @@ export async function generateFlowLockInternal(
folder: string,
dryRun: boolean,
workspace: Workspace,
opts: GlobalOptions & {
defaultTs?: "bun" | "deno";
},
justUpdateMetadataLock?: boolean,
noStaleMessage?: boolean
): Promise<string | undefined> {
noStaleMessage?: boolean,
useRawReqs?: boolean,
): Promise<string | void> {
if (folder.endsWith(SEP)) {
folder = folder.substring(0, folder.length - 1);
}
@@ -114,7 +104,24 @@ export async function generateFlowLockInternal(
log.info(`Generating lock for flow ${folder} at ${remote_path}`);
}
let hashes = await generateFlowHash(folder);
let rawReqs: Record<string, string> | undefined = undefined;
if (useRawReqs) {
// Find all dependency files in the workspace
const globalDeps = await findGlobalDeps();
// Find closest dependency files for this flow
rawReqs = {};
// TODO: PERF: Only include raw reqs for the languages that are in the flow
languagesWithRawReqsSupport.map((lang) => {
const dep = findClosestRawReqs(lang, folder, globalDeps);
if (dep) {
// @ts-ignore
rawReqs[lang.language] = dep;
}
});
}
let hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs);
const conf = await readLockfile();
if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) {
@@ -128,8 +135,17 @@ export async function generateFlowLockInternal(
return remote_path;
}
if (useRawReqs) {
log.warn("If using local lockfiles, following redeployments from Web App will inevitably override generated lockfiles by CLI. To maintain your script's lockfiles you will need to redeploy only from CLI. (Behavior is subject to change)")
log.info(
(await blueColor())(
`Found raw requirements (${languagesWithRawReqsSupport.map((l) => l.rrFilename).join("/")}) for ${folder}, using it`,
),
);
}
const flowValue = (await yamlParseFile(
folder! + SEP + "flow.yaml"
folder! + SEP + "flow.yaml",
)) as FlowFile;
if (!justUpdateMetadataLock) {
@@ -148,26 +164,40 @@ export async function generateFlowLockInternal(
replaceInlineScripts(
flowValue.value.modules,
folder + SEP!,
changedScripts
changedScripts,
);
//removeChangedLocks
flowValue.value = await updateFlow(workspace, flowValue.value, remote_path);
flowValue.value = await updateFlow(
workspace,
flowValue.value,
remote_path,
rawReqs,
);
const inlineScripts = extractInlineScriptsForFlows(
flowValue.value.modules,
newPathAssigner("bun")
newPathAssigner(opts.defaultTs ?? "bun"),
);
inlineScripts
.filter((s) => s.path.endsWith(".lock"))
.forEach((s) => {
Deno.writeTextFileSync(
Deno.cwd() + SEP + folder + SEP + s.path,
s.content
s.content,
);
});
// Overwrite `flow.yaml` with the new lockfile references
await Deno.writeTextFile(
Deno.cwd() + SEP + folder + SEP + "flow.yaml",
yamlStringify(
flowValue as Record<string, any>
)
);
}
hashes = await generateFlowHash(folder);
hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs);
for (const [path, hash] of Object.entries(hashes)) {
await updateMetadataGlobalLock(folder, hash, path);
@@ -176,7 +206,7 @@ export async function generateFlowLockInternal(
}
// on windows, when using powershell, blue is not readable
async function blueColor(): Promise<(x: string) => void> {
export async function blueColor(): Promise<(x: string) => void> {
const isWin = await getIsWin();
return isWin ? colors.black : colors.blue;
}
@@ -193,7 +223,7 @@ export async function generateScriptMetadataInternal(
noStaleMessage: boolean,
globalDeps: GlobalDeps,
codebases: SyncCodebase[],
justUpdateMetadataLock?: boolean
justUpdateMetadataLock?: boolean,
): Promise<string | undefined> {
const remotePath = scriptPath
.substring(0, scriptPath.indexOf("."))
@@ -201,23 +231,26 @@ export async function generateScriptMetadataInternal(
const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs);
const rrLang = languagesWithRawReqsSupport.find((l) => language == l.language);
const rawReqs = findClosestRawReqs(
language as "bun" | "python3" | "php" | "go" | undefined,
rrLang,
scriptPath,
globalDeps
globalDeps,
);
if (rawReqs) {
if (rawReqs && rrLang) {
log.info(
(await blueColor())(
`Found raw requirements (package.json/requirements.txt/composer.json/go.mod) for ${scriptPath}, using it`
)
`Found raw requirements (${rrLang.rrFilename}) for ${scriptPath}, using it`,
),
);
}
const metadataWithType = await parseMetadataFile(
remotePath,
undefined,
globalDeps,
codebases
codebases,
);
// read script content
@@ -229,7 +262,7 @@ export async function generateScriptMetadataInternal(
if (await checkifMetadataUptodate(remotePath, hash, undefined)) {
if (!noStaleMessage) {
log.info(
colors.green(`Script ${remotePath} metadata is up-to-date, skipping`)
colors.green(`Script ${remotePath} metadata is up-to-date, skipping`),
);
}
return;
@@ -251,7 +284,7 @@ export async function generateScriptMetadataInternal(
scriptContent,
language,
metadataParsedContent,
scriptPath
scriptPath,
);
}
@@ -265,7 +298,7 @@ export async function generateScriptMetadataInternal(
language,
remotePath,
metadataParsedContent,
rawReqs
rawReqs,
);
} else {
metadataParsedContent.lock = "";
@@ -286,7 +319,7 @@ export async function generateScriptMetadataInternal(
hash = await generateScriptHash(
rawReqs,
scriptContent,
metadataContentUsedForHash
metadataContentUsedForHash,
);
await updateMetadataGlobalLock(remotePath, hash);
if (!justUpdateMetadataLock) {
@@ -299,14 +332,14 @@ export async function updateScriptSchema(
scriptContent: string,
language: ScriptLanguage,
metadataContent: Record<string, any>,
path: string
path: string,
): Promise<void> {
// infer schema from script content and update it inplace
const result = await inferSchema(
language,
scriptContent,
metadataContent.schema,
path
path,
);
metadataContent.schema = result.schema;
if (result.has_preprocessor) {
@@ -327,15 +360,12 @@ async function updateScriptLock(
language: ScriptLanguage,
remotePath: string,
metadataContent: Record<string, any>,
rawDeps: string | undefined
rawDeps: string | undefined,
): Promise<void> {
if (
!(
language == "bun" ||
language == "python3" ||
language == "go" ||
languagesWithRawReqsSupport.some((l) => l.language == language) ||
language == "deno" ||
language == "php" ||
language == "rust" ||
language == "ansible"
)
@@ -363,7 +393,7 @@ async function updateScriptLock(
raw_deps: rawDeps,
entrypoint: remotePath,
}),
}
},
);
let responseText = "reading response failed";
@@ -374,11 +404,11 @@ async function updateScriptLock(
if (lock === undefined) {
if (response?.["error"]?.["message"]) {
throw new LockfileGenerationError(
`Failed to generate lockfile: ${response?.["error"]?.["message"]}`
`Failed to generate lockfile: ${response?.["error"]?.["message"]}`,
);
}
throw new LockfileGenerationError(
`Failed to generate lockfile: ${JSON.stringify(response, null, 2)}`
`Failed to generate lockfile: ${JSON.stringify(response, null, 2)}`,
);
}
const lockPath = remotePath + ".script.lock";
@@ -390,7 +420,7 @@ async function updateScriptLock(
if (await Deno.stat(lockPath)) {
await Deno.remove(lockPath);
}
} catch {}
} catch { }
metadataContent.lock = "";
}
} catch (e) {
@@ -398,7 +428,7 @@ async function updateScriptLock(
throw e;
}
throw new LockfileGenerationError(
`Failed to generate lockfile:${rawResponse.statusText}, ${responseText}, ${e}`
`Failed to generate lockfile:${rawResponse.statusText}, ${responseText}, ${e}`,
);
}
}
@@ -406,24 +436,48 @@ async function updateScriptLock(
export async function updateFlow(
workspace: Workspace,
flow_value: FlowValue,
remotePath: string
remotePath: string,
rawDeps?: Record<string, string>
): Promise<FlowValue | undefined> {
// generate the script lock running a dependency job in Windmill and update it inplace
// TODO: update this once the client is released
const rawResponse = await fetch(
`${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`,
{
method: "POST",
headers: {
Cookie: `token=${workspace.token}`,
"Content-Type": "application/json",
let rawResponse;
if (rawDeps != undefined) {
log.info(colors.blue("Using raw requirements for flow dependencies"));
// generate the script lock running a dependency job in Windmill and update it inplace
rawResponse = await fetch(
`${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`,
{
method: "POST",
headers: {
Cookie: `token=${workspace.token}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
flow_value,
path: remotePath,
use_local_lockfiles: true,
raw_deps: rawDeps,
}),
},
body: JSON.stringify({
flow_value,
path: remotePath,
}),
}
);
);
} else {
// Standard dependency resolution on the server
rawResponse = await fetch(
`${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`,
{
method: "POST",
headers: {
Cookie: `token=${workspace.token}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
flow_value,
path: remotePath,
}),
},
);
}
let responseText = "reading response failed";
try {
@@ -435,20 +489,20 @@ export async function updateFlow(
const msg = (res as any)?.["error"]?.["message"];
if (msg) {
throw new LockfileGenerationError(
`Failed to generate lockfile: ${msg}`
`Failed to generate lockfile: ${msg}`,
);
}
throw new LockfileGenerationError(
`Failed to generate lockfile: ${rawResponse.statusText}, ${responseText}`
`Failed to generate lockfile: ${rawResponse.statusText}, ${responseText}`,
);
}
return (res as any).updated_flow_value;
} catch (e) {
try {
responseText = await rawResponse.text();
} catch {}
} catch { }
throw new Error(
`Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`
`Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`,
);
}
}
@@ -460,7 +514,7 @@ export async function inferSchema(
language: ScriptLanguage,
content: string,
currentSchema: any,
path: string
path: string,
): Promise<{
schema: any;
has_preprocessor: boolean | undefined;
@@ -586,8 +640,8 @@ export async function inferSchema(
if (inferedSchema.type == "Invalid") {
log.info(
colors.yellow(
`Script ${path} invalid, it cannot be parsed to infer schema.`
)
`Script ${path} invalid, it cannot be parsed to infer schema.`,
),
);
return {
schema: defaultScriptMetadata().schema,
@@ -601,7 +655,7 @@ export async function inferSchema(
}
currentSchema.required = [];
const oldProperties = JSON.parse(
JSON.stringify(currentSchema?.properties ?? {})
JSON.stringify(currentSchema?.properties ?? {}),
);
currentSchema.properties = {};
@@ -612,7 +666,7 @@ export async function inferSchema(
currentSchema.properties[arg.name] = oldProperties[arg.name];
}
currentSchema.properties[arg.name] = sortObject(
currentSchema.properties[arg.name]
currentSchema.properties[arg.name],
);
argSigToJsonSchemaType(arg.typ, currentSchema.properties[arg.name]);
@@ -639,7 +693,7 @@ function sortObject(obj: any): any {
...acc,
[key]: obj[key],
}),
{}
{},
);
}
@@ -649,24 +703,24 @@ export function argSigToJsonSchemaType(
| string
| { resource: string | null }
| {
list:
| (string | { object: { key: string; typ: any }[] })
| { str: any }
| { object: { key: string; typ: any }[] }
| null;
}
list:
| (string | { object: { key: string; typ: any }[] })
| { str: any }
| { object: { key: string; typ: any }[] }
| null;
}
| { dynselect: string }
| { str: string[] | null }
| { object: { key: string; typ: any }[] }
| {
oneof: [
{
label: string;
properties: { key: string; typ: any }[];
}
];
},
oldS: SchemaProperty
oneof: [
{
label: string;
properties: { key: string; typ: any }[];
},
];
},
oldS: SchemaProperty,
): void {
const newS: SchemaProperty = { type: "" };
if (t === "int") {
@@ -858,7 +912,9 @@ export function replaceLock(o?: { lock?: string | string[] }) {
o.lock = readInlinePathSync(lockPath);
} catch (e) {
log.info(
colors.yellow(`Failed to read lockfile, doing as if it was empty: ${e}`)
colors.yellow(
`Failed to read lockfile, doing as if it was empty: ${e}`,
),
);
o.lock = "";
}
@@ -868,13 +924,13 @@ export async function parseMetadataFile(
scriptPath: string,
generateMetadataIfMissing:
| (GlobalOptions & {
path: string;
workspaceRemote: Workspace;
schemaOnly?: boolean;
})
path: string;
workspaceRemote: Workspace;
schemaOnly?: boolean;
})
| undefined,
globalDeps: GlobalDeps,
codebases: SyncCodebase[]
codebases: SyncCodebase[],
): Promise<{ isJson: boolean; payload: any; path: string }> {
let metadataFilePath = scriptPath + ".script.json";
try {
@@ -900,14 +956,14 @@ export async function parseMetadataFile(
// no metadata file at all. Create it
log.info(
(await blueColor())(
`Creating script metadata file for ${metadataFilePath}`
)
`Creating script metadata file for ${metadataFilePath}`,
),
);
metadataFilePath = scriptPath + ".script.yaml";
let scriptInitialMetadata = defaultScriptMetadata();
const scriptInitialMetadataYaml = yamlStringify(
scriptInitialMetadata as Record<string, any>,
yamlOptions
yamlOptions,
);
await Deno.writeTextFile(metadataFilePath, scriptInitialMetadataYaml, {
createNew: true,
@@ -916,8 +972,8 @@ export async function parseMetadataFile(
if (generateMetadataIfMissing) {
log.info(
(await blueColor())(
`Generating lockfile and schema for ${metadataFilePath}`
)
`Generating lockfile and schema for ${metadataFilePath}`,
),
);
try {
await generateScriptMetadataInternal(
@@ -928,10 +984,10 @@ export async function parseMetadataFile(
false,
globalDeps,
codebases,
false
false,
);
scriptInitialMetadata = (await yamlParseFile(
metadataFilePath
metadataFilePath,
)) as ScriptMetadata;
if (!generateMetadataIfMissing.schemaOnly) {
replaceLock(scriptInitialMetadata);
@@ -939,8 +995,8 @@ export async function parseMetadataFile(
} catch (e) {
log.info(
colors.yellow(
`Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`
)
`Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`,
),
);
}
}
@@ -979,7 +1035,7 @@ export async function checkifMetadataUptodate(
path: string,
hash: string,
conf: Lock | undefined,
subpath?: string
subpath?: string,
) {
if (!conf) {
conf = await readLockfile();
@@ -995,17 +1051,17 @@ export async function checkifMetadataUptodate(
export async function generateScriptHash(
rawReqs: string | undefined,
scriptContent: string,
newMetadataContent: string
newMetadataContent: string,
) {
return await generateHash(
(rawReqs ?? "") + scriptContent + newMetadataContent
(rawReqs ?? "") + scriptContent + newMetadataContent,
);
}
export async function updateMetadataGlobalLock(
path: string,
hash: string,
subpath?: string
subpath?: string,
): Promise<void> {
const conf = await readLockfile();
if (!conf?.locks) {
@@ -1024,6 +1080,6 @@ export async function updateMetadataGlobalLock(
}
await Deno.writeTextFile(
WMILL_LOCKFILE,
yamlStringify(conf as Record<string, any>, yamlOptions)
yamlStringify(conf as Record<string, any>, yamlOptions),
);
}
cli/resource-type.ts
+2 -1
View File
@@ -5,7 +5,8 @@ import {
parseFromFile,
removeType,
} from "./types.ts";
import { requireLogin, resolveWorkspace } from "./context.ts";
import { requireLogin } from "./auth.ts";
import { resolveWorkspace } from "./context.ts";
import { colors, Command, log, Table } from "./deps.ts";
import * as wmill from "./gen/services.gen.ts";
import { ResourceType } from "./gen/types.gen.ts";
cli/resource.ts
+2 -1
View File
@@ -5,7 +5,8 @@ import {
parseFromFile,
removeType,
} from "./types.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import { requireLogin } from "./auth.ts";
import { resolveWorkspace, validatePath } from "./context.ts";
import { colors, Command, log, SEP, Table } from "./deps.ts";
import * as wmill from "./gen/services.gen.ts";
import { Resource } from "./gen/types.gen.ts";
cli/schedule.ts
+2 -1
View File
@@ -1,6 +1,7 @@
// deno-lint-ignore-file no-explicit-any
import { colors, Command, log, SEP, Table } from "./deps.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import { requireLogin } from "./auth.ts";
import { resolveWorkspace, validatePath } from "./context.ts";
import * as wmill from "./gen/services.gen.ts";
import {
cli/script.ts
+21 -26
View File
@@ -1,6 +1,7 @@
// deno-lint-ignore-file no-explicit-any
import { GlobalOptions } from "./types.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import { requireLogin } from "./auth.ts";
import { resolveWorkspace, validatePath } from "./context.ts";
import {
colors,
Command,
@@ -26,8 +27,10 @@ import {
parseMetadataFile,
} from "./metadata.ts";
import {
LanguageWithRawReqsSupport,
ScriptLanguage,
inferContentTypeFromFilePath,
languagesWithRawReqsSupport,
} from "./script_common.ts";
import {
elementsToMap,
@@ -865,43 +868,35 @@ async function bootstrap(
);
}
export type GlobalDeps = {
pkgs: Record<string, string>;
reqs: Record<string, string>;
composers: Record<string, string>;
goMods: Record<string, string>;
};
export type GlobalDeps = Map<LanguageWithRawReqsSupport, Record<string, string>>;
export async function findGlobalDeps(): Promise<GlobalDeps> {
const pkgs: { [key: string]: string } = {};
const reqs: { [key: string]: string } = {};
const composers: { [key: string]: string } = {};
const goMods: { [key: string]: string } = {};
var globalDeps: GlobalDeps = new Map();
const els = await FSFSElement(Deno.cwd(), [], false);
for await (const entry of readDirRecursiveWithIgnore((p, isDir) => {
p = SEP + p;
return (
!isDir &&
!(
p.endsWith(SEP + "package.json") ||
p.endsWith(SEP + "requirements.txt") ||
p.endsWith(SEP + "composer.json") ||
p.endsWith(SEP + "go.mod")
// Skip if the filename is not one of lockfile names
!(languagesWithRawReqsSupport.some(
lockfile =>
p.endsWith(SEP + lockfile.rrFilename))
)
);
}, els)) {
if (entry.isDirectory || entry.ignored) continue;
const content = await entry.getContentText();
if (entry.path.endsWith("package.json")) {
pkgs[entry.path.substring(0, entry.path.length - "package.json".length)] = content;
} else if (entry.path.endsWith("requirements.txt")) {
reqs[entry.path.substring(0, entry.path.length - "requirements.txt".length)] = content;
} else if (entry.path.endsWith("composer.json")) {
composers[entry.path.substring(0, entry.path.length - "composer.json".length)] = content;
} else if (entry.path.endsWith("go.mod")) {
goMods[entry.path.substring(0, entry.path.length - "go.mod".length)] = content;
}
// Iterate over available languages to find which lockfile
languagesWithRawReqsSupport.map((lock) => {
if (entry.path.endsWith(lock.rrFilename)){
const current = globalDeps.get(lock) ?? {};
current[entry.path.substring(0, entry.path.length - lock.rrFilename.length)] = content;
globalDeps.set(lock, current);
}
});
}
return { pkgs, reqs, composers, goMods };
return globalDeps;
}
async function generateMetadata(
opts: GlobalOptions & {
cli/script_common.ts
+17
View File
@@ -22,6 +22,23 @@ export type ScriptLanguage =
| "java";
// for related places search: ADD_NEW_LANG
// To make language support raw requirements:
// 1. Add value here
// 2. Modify backend to allow raw deps
export type LanguageWithRawReqsSupport =
| { language: "bun", rrFilename /** (raw requirements filename) */: "package.json" }
// TODO: Add `requirements.in` - more intuitive and reflects better what actually happens
| { language: "python3", rrFilename: "requirements.txt" }
| { language: "php", rrFilename: "composer.json" }
| { language: "go", rrFilename: "go.mod" };
export const languagesWithRawReqsSupport: LanguageWithRawReqsSupport[] = [
{ language: "bun", rrFilename: "package.json" },
{ language: "python3", rrFilename: "requirements.txt" },
{ language: "php", rrFilename: "composer.json" },
{ language: "go", rrFilename: "go.mod" },
] as const;
export function inferContentTypeFromFilePath(
contentPath: string,
defaultTs: "bun" | "deno" | undefined
cli/sync.ts
+9 -40
View File
@@ -1,4 +1,5 @@
import { fetchVersion, requireLogin, resolveWorkspace } from "./context.ts";
import { requireLogin } from "./auth.ts";
import { fetchVersion, resolveWorkspace } from "./context.ts";
import {
colors,
Command,
@@ -55,43 +56,8 @@ function mergeCliWithEffectiveOptions<T extends GlobalOptions & SyncOptions & {
cliOpts: T,
effectiveOpts: SyncOptions
): T {
// Start with effective options from config, then overlay only explicitly provided CLI flags
const mergedOpts = Object.assign({}, effectiveOpts) as T;
// Always preserve these operational CLI flags
if (cliOpts.dryRun !== undefined) mergedOpts.dryRun = cliOpts.dryRun;
if (cliOpts.yes !== undefined) mergedOpts.yes = cliOpts.yes;
if (cliOpts.stateful !== undefined) mergedOpts.stateful = cliOpts.stateful;
if (cliOpts.skipPull !== undefined) mergedOpts.skipPull = cliOpts.skipPull;
if (cliOpts.failConflicts !== undefined) mergedOpts.failConflicts = cliOpts.failConflicts;
if (cliOpts.plainSecrets !== undefined) mergedOpts.plainSecrets = cliOpts.plainSecrets;
if (cliOpts.json !== undefined) mergedOpts.json = cliOpts.json;
if (cliOpts.message !== undefined) mergedOpts.message = cliOpts.message;
if (cliOpts.parallel !== undefined) mergedOpts.parallel = cliOpts.parallel;
if (cliOpts.jsonOutput !== undefined) mergedOpts.jsonOutput = cliOpts.jsonOutput;
if (cliOpts.repository !== undefined) mergedOpts.repository = cliOpts.repository;
// Always preserve CLI include flags (they should override config file settings)
if (cliOpts.includeUsers !== undefined) mergedOpts.includeUsers = cliOpts.includeUsers;
if (cliOpts.includeGroups !== undefined) mergedOpts.includeGroups = cliOpts.includeGroups;
if (cliOpts.includeSettings !== undefined) mergedOpts.includeSettings = cliOpts.includeSettings;
if (cliOpts.includeKey !== undefined) mergedOpts.includeKey = cliOpts.includeKey;
if (cliOpts.includeSchedules !== undefined) mergedOpts.includeSchedules = cliOpts.includeSchedules;
if (cliOpts.includeTriggers !== undefined) mergedOpts.includeTriggers = cliOpts.includeTriggers;
// Always preserve CLI skip flags (they should override config file settings)
if (cliOpts.skipScripts !== undefined) mergedOpts.skipScripts = cliOpts.skipScripts;
if (cliOpts.skipFolders !== undefined) mergedOpts.skipFolders = cliOpts.skipFolders;
if (cliOpts.skipVariables !== undefined) mergedOpts.skipVariables = cliOpts.skipVariables;
if (cliOpts.skipResources !== undefined) mergedOpts.skipResources = cliOpts.skipResources;
if (cliOpts.skipResourceTypes !== undefined) mergedOpts.skipResourceTypes = cliOpts.skipResourceTypes;
if (cliOpts.skipSecrets !== undefined) mergedOpts.skipSecrets = cliOpts.skipSecrets;
if (cliOpts.skipFlows !== undefined) mergedOpts.skipFlows = cliOpts.skipFlows;
if (cliOpts.skipApps !== undefined) mergedOpts.skipApps = cliOpts.skipApps;
return mergedOpts;
// overlay CLI options on top (undefined cliOpts won't override effectiveOpts)
return Object.assign({}, effectiveOpts, cliOpts) as T;
}
// Resolve effective sync options with smart repository detection
@@ -163,12 +129,14 @@ async function resolveEffectiveSyncOptions(
} else {
// Non-interactive mode - list options and use top-level
log.warn(`Multiple repository overrides found: ${applicableRepos.join(', ')}`);
log.warn(`Use --repository flag to specify which one to use. Using top-level settings.`);
log.warn(`Running in non-interactive mode. Use --repository flag to specify which one to use.`);
log.info(`Falling back to top-level settings (no repository-specific overrides applied)`);
}
}
}
// No repository overrides found or selected - use top-level settings
log.info(`No repository overrides found, using top-level settings`);
return getEffectiveSettings(
localConfig,
workspace.remote,
@@ -1491,7 +1459,7 @@ export async function pull(opts: GlobalOptions & SyncOptions & { repository?: st
}
for (const change of tracker.flows) {
log.info(`Updating lock for flow ${change}`);
await generateFlowLockInternal(change, false, workspace, true);
await generateFlowLockInternal(change, false, workspace, opts, true);
}
if (tracker.apps.length > 0) {
log.info(
@@ -1686,6 +1654,7 @@ export async function push(opts: GlobalOptions & SyncOptions & { repository?: st
change,
true,
workspace,
opts,
false,
true
);
cli/test/gitsync_settings_features.test.ts
+8 -8
View File
@@ -55,7 +55,7 @@ excludes: []`);
// Read updated config
const updatedConfig = await Deno.readTextFile(`${tempDir}/wmill.yaml`);
const backendUrl = new URL(backend.baseUrl).toString();
// Should create workspace wildcard override
assertStringIncludes(updatedConfig, `'${backendUrl}:${backend.workspace}:*':`);
assertStringIncludes(updatedConfig, "overrides:");
@@ -109,14 +109,14 @@ skipVariables: false`);
// Read updated config
const updatedConfig = await Deno.readTextFile(`${tempDir}/wmill.yaml`);
// Should update top-level settings, not create overrides
assertStringIncludes(updatedConfig, "includes:\n - f/special/**");
assertStringIncludes(updatedConfig, "excludes:\n - '*.test.ts'");
assertStringIncludes(updatedConfig, "extraIncludes:\n - g/**");
// Should NOT have overrides section
assertEquals(updatedConfig.includes("overrides:"), false, "Default mode should not create overrides");
// Should have empty overrides section for consistency
assertStringIncludes(updatedConfig, "overrides: {}", "Should have empty overrides section for consistency");
});
});
@@ -168,10 +168,10 @@ skipResources: false`);
], tempDir);
assertEquals(result.code, 0);
// Should show differences
assertStringIncludes(result.stdout, "Changes that would be made:");
assertStringIncludes(result.stdout, "Changes that would be applied locally:");
// Should show the change for skipResources (ignoring ANSI color codes)
assertStringIncludes(result.stdout, "skipResources:");
});
});
});
cli/test/init_no_git_sync.test.ts
+3 -3
View File
@@ -126,8 +126,8 @@ Deno.test("Init: --use-backend flag applies git-sync settings", async () => {
assertStringIncludes(wmillYaml, "*.test.ts", "Should include backend's exclude_path");
assertStringIncludes(wmillYaml, "g/**", "Should include backend's extra_include_path");
// Should NOT have overrides section since we're starting fresh
assertEquals(wmillYaml.includes("overrides:"), false, "Init should not create overrides section");
// Should have empty overrides section for consistency
assertStringIncludes(wmillYaml, "overrides: {}");
});
});
@@ -180,6 +180,6 @@ Deno.test("Init: --use-default bypasses backend settings check", async () => {
// Should NOT have backend-specific settings
assertEquals(wmillYaml.includes("f/should-be-ignored/**"), false, "Should not include backend settings");
assertEquals(wmillYaml.includes("overrides:"), false, "Should not create overrides when using defaults");
assertStringIncludes(wmillYaml, "overrides: {}", "Should have empty overrides section for consistency");
});
});

Some files were not shown because too many files have changed in this diff Show More