diff --git a/.github/workflows/backend-check.yml b/.github/workflows/backend-check.yml index 338e1c2008..11f8af5c29 100644 --- a/.github/workflows/backend-check.yml +++ b/.github/workflows/backend-check.yml @@ -20,7 +20,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend timeout-minutes: 16 @@ -41,13 +41,13 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend timeout-minutes: 16 run: | mkdir -p fake_frontend_build - FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features $(./all_features_oss.sh) + FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features all_sqlx_features check_ee: runs-on: ubicloud-standard-8 @@ -75,7 +75,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend timeout-minutes: 16 @@ -112,10 +112,10 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check timeout-minutes: 16 working-directory: ./backend run: | mkdir -p fake_frontend_build - FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --all-features + FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features all_sqlx_features,private diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index f46d255320..c61f607e17 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -33,7 +33,7 @@ jobs: - name: Setup Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.88.0 + toolchain: 1.90.0 override: true - name: Substitute EE code diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index e5b7ba1b41..39e3de253c 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -82,7 +82,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend - toolchain: 1.88.0 + toolchain: 1.90.0 - name: cargo check working-directory: ./backend diff --git a/.github/workflows/git-commands.yaml b/.github/workflows/git-commands.yaml index 8a5d94df64..6307a341b3 100644 --- a/.github/workflows/git-commands.yaml +++ b/.github/workflows/git-commands.yaml @@ -117,6 +117,85 @@ jobs: body: 'Successfully ran sqlx update' }) + demo: + if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/demo') + runs-on: ubicloud-standard-2 + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run Claude Code for Demo Generation + uses: anthropics/claude-code-action@beta + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + timeout_minutes: "10" + allowed_tools: "Bash" + direct_prompt: | + You need to: + + 1. Extract the Cloudflare preview URL from the cloudflare-workers-and-pages bot comment in this PR + 2. Analyze the PR changes to understand what feature was added/modified + 3. Create detailed demo instructions showing how to showcase the new feature + 4. Create a demo.json file with a valid JSON object containing: + - instructions: the demo instructions + - url: the preview URL + 5. VALIDATE the JSON file using `jq` before finishing + DO NOT COMMIT THIS FILE TO THE PR. + + Example demo.json: + { + "instructions": "Instructions here", + "url": "https://example.pages.dev" + } + + CRITICAL: After creating demo.json, you MUST: + 1. Run `jq empty demo.json` to validate the JSON is properly formatted + 2. If validation fails, fix the JSON and validate again + 3. Only proceed once the JSON passes validation + 4. Use proper JSON escaping for newlines, quotes, and special characters + + Make sure to: + - Create a valid JSON object that passes `jq empty demo.json` + - Extract the correct preview URL (should be a .pages.dev domain) + - Create specific, actionable demo steps based on the actual changes in the PR + - Properly escape all strings in the JSON (use jq to create the file if needed) + - NOT COMMIT THE DEMO.JSON FILE TO THE PR + + - name: Send instructions to Windmill + env: + DEMO_WEBHOOK_TOKEN: ${{ secrets.DEMO_WEBHOOK_TOKEN }} + run: | + if [[ -f "demo.json" ]]; then + echo "Found demo.json, sending to Windmill..." + cat demo.json + + # Validate JSON one more time (Claude should have already done this) + if ! jq empty demo.json; then + echo "Error: demo.json is not valid JSON" + exit 1 + fi + + RESULT=$(curl -s \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $DEMO_WEBHOOK_TOKEN" \ + -X POST \ + -d @demo.json \ + 'https://app.windmill.dev/api/w/windmill-labs/jobs/run/f/f/ai/browserbase_demo') + + echo "Windmill response:" + echo -E "$RESULT" + else + echo "Error: demo.json file not found" + exit 1 + fi + update-ee-ref: if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/eeref') runs-on: ubicloud-standard-2 diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index a36eca7d85..95545c14a1 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -35,7 +35,7 @@ jobs: - name: Setup Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.88.0 + toolchain: 1.90.0 override: true - name: Substitute EE code diff --git a/CHANGELOG.md b/CHANGELOG.md index ff16a84675..ad25a9c096 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [1.545.0](https://github.com/windmill-labs/windmill/compare/v1.544.2...v1.545.0) (2025-09-20) + + +### Features + +* force cancel in batch cancel ([2eeaf56](https://github.com/windmill-labs/windmill/commit/2eeaf5639dfcdb0114dc5fc323dbd10ef5ceaca5)) +* load for loop jobs timeline directly from for loop flow status ([#6646](https://github.com/windmill-labs/windmill/issues/6646)) ([f71f9b0](https://github.com/windmill-labs/windmill/commit/f71f9b089438fc29dfb26a21ec2b8aa5867d0296)) + + +### Bug Fixes + +* add termination handler earlier in lifecycle ([56cdb69](https://github.com/windmill-labs/windmill/commit/56cdb69e599e295a51284c58d6ff56498ad22fd7)) +* allow variable picker in password field ([194887e](https://github.com/windmill-labs/windmill/commit/194887e97b3250ef28b0ddf5c03932148858ca24)) +* fix flow quick picker stuck ([#6638](https://github.com/windmill-labs/windmill/issues/6638)) ([5cab802](https://github.com/windmill-labs/windmill/commit/5cab802c421c5aeda8c2b57242a4e0c4c074a5f8)) +* fix too strict aggrid coldef validation ([612d003](https://github.com/windmill-labs/windmill/commit/612d00367c2a288920c14449561433d199b95cae)) +* retry python relative imports on errno 104 ([81ff0dc](https://github.com/windmill-labs/windmill/commit/81ff0dcd8cc2fb1c5fc12a558702cbc950ca1eb4)) +* teams api improvements ([#6643](https://github.com/windmill-labs/windmill/issues/6643)) ([70e9ae1](https://github.com/windmill-labs/windmill/commit/70e9ae14a9541862aaa31b8435abb0185805d4b7)) + ## [1.544.2](https://github.com/windmill-labs/windmill/compare/v1.544.1...v1.544.2) (2025-09-18) diff --git a/Dockerfile b/Dockerfile index e8a807986a..5eb92277bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG DEBIAN_IMAGE=debian:bookworm-slim -ARG RUST_IMAGE=rust:1.88-slim-bookworm +ARG RUST_IMAGE=rust:1.90-slim-bookworm # Build libwindmill_duckdb_ffi_internal.so separately FROM ${RUST_IMAGE} AS windmill_duckdb_ffi_internal_builder diff --git a/backend/.sqlx/query-0088b5bf5cf1c7e47f18c3d05cce09f3462c8e39cb3291f0d497742ad4763fa3.json b/backend/.sqlx/query-0088b5bf5cf1c7e47f18c3d05cce09f3462c8e39cb3291f0d497742ad4763fa3.json deleted file mode 100644 index 5fa8d88b2e..0000000000 --- a/backend/.sqlx/query-0088b5bf5cf1c7e47f18c3d05cce09f3462c8e39cb3291f0d497742ad4763fa3.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET flow_status = \n CASE WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT],\n $4\n ),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $5),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $6)\n ELSE\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4)\n END\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Jsonb", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "0088b5bf5cf1c7e47f18c3d05cce09f3462c8e39cb3291f0d497742ad4763fa3" -} diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index d29a18c691..e7ed0aee65 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - true, - true, - true, - true, - true, + false, + false, + false, + false, + false, true, true ] diff --git a/backend/.sqlx/query-081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f.json b/backend/.sqlx/query-081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f.json new file mode 100644 index 0000000000..75f021b168 --- /dev/null +++ b/backend/.sqlx/query-081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET flow_status = \n CASE WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status,\n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT],\n $4\n ),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $5),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $6)\n ELSE\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4)\n END\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "081f838b3dbe81631d17e7ca0751db725a7f92d4e43a86bcfa06a4ac7c70ac8f" +} diff --git a/backend/.sqlx/query-0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c.json b/backend/.sqlx/query-0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c.json new file mode 100644 index 0000000000..fbe7e2dec1 --- /dev/null +++ b/backend/.sqlx/query-0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "VACUUM (SKIP_LOCKED) v2_job_queue, v2_job_runtime, v2_job_status, job_perms", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0cc221cb8b3059b21e6b3b4c874b8f4d32815edd2090ccb5d562a89142a7dd9c" +} diff --git a/backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json b/backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json deleted file mode 100644 index 1af42ff529..0000000000 --- a/backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "hash", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f" -} diff --git a/backend/.sqlx/query-2d0df658c31d45592dca7add8c69d06da7d81e7177a324e3f3f9af1076b3ab66.json b/backend/.sqlx/query-2d0df658c31d45592dca7add8c69d06da7d81e7177a324e3f3f9af1076b3ab66.json deleted file mode 100644 index 9deb40619a..0000000000 --- a/backend/.sqlx/query-2d0df658c31d45592dca7add8c69d06da7d81e7177a324e3f3f9af1076b3ab66.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET flow_status = \n CASE \n WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n )\n ELSE\n JSONB_SET(JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n ) END\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "2d0df658c31d45592dca7add8c69d06da7d81e7177a324e3f3f9af1076b3ab66" -} diff --git a/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json b/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json deleted file mode 100644 index 6973159764..0000000000 --- a/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "VACUUM v2_job_queue, v2_job_runtime, v2_job_status, job_perms", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc" -} diff --git a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json deleted file mode 100644 index 6f08d98113..0000000000 --- a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar", - "Varchar", - "Int8", - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - }, - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf" -} diff --git a/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json b/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json deleted file mode 100644 index bce7324fb6..0000000000 --- a/backend/.sqlx/query-85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag) SELECT unnest($1::uuid[]), $2, now(), $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "85705fc3d7f8ba5f1b12d5fb222c38fc64deb1226aab9dc3bc4465324fce37d1" -} diff --git a/backend/.sqlx/query-93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa.json b/backend/.sqlx/query-93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa.json new file mode 100644 index 0000000000..7600acb32a --- /dev/null +++ b/backend/.sqlx/query-93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET flow_status = \n CASE \n WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n ELSE\n JSONB_SET(JSONB_SET(\n flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n END\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "93f00e2d164c090a959da12cbb7e4fc6e5804fea077116d339a041c0f2b0c1fa" +} diff --git a/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json b/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json deleted file mode 100644 index 1afc61978e..0000000000 --- a/backend/.sqlx/query-a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_status (id, flow_status) SELECT unnest($1::uuid[]), $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "a0b3e10e077d30c1da135dff9feca3761d400391f1f46a8294da3e6c9af63887" -} diff --git a/backend/.sqlx/query-a49e59c814d440bfed7d6bfc2755518225b1243a9d39ce659b95637c33d0954f.json b/backend/.sqlx/query-a49e59c814d440bfed7d6bfc2755518225b1243a9d39ce659b95637c33d0954f.json deleted file mode 100644 index 1315df082f..0000000000 --- a/backend/.sqlx/query-a49e59c814d440bfed7d6bfc2755518225b1243a9d39ce659b95637c33d0954f.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET flow_status = \n CASE \n WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN\n JSONB_SET(\n JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), \n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n ELSE\n JSONB_SET(JSONB_SET(\n flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'],\n ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb\n )\n END\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int4", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Text", - "Jsonb", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "a49e59c814d440bfed7d6bfc2755518225b1243a9d39ce659b95637c33d0954f" -} diff --git a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json b/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json deleted file mode 100644 index e07a3ccdd7..0000000000 --- a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id, raw_flow) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, 1)) RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Int8", - "Varchar", - { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlescriptflow", - "flowscript", - "flownode", - "appscript", - "aiagent" - ] - } - } - }, - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - }, - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Jsonb" - ] - }, - "nullable": [ - false - ] - }, - "hash": "ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338" -} diff --git a/backend/.sqlx/query-ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045.json b/backend/.sqlx/query-ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045.json new file mode 100644 index 0000000000..8de90d69d7 --- /dev/null +++ b/backend/.sqlx/query-ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE v2_job_status SET flow_status = \n JSONB_SET(JSONB_SET(JSONB_SET(JSONB_SET(\n flow_status, \n ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), \n ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5),\n ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6)\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int4", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid", + "Text", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ac5dd4a4d7991159e053f1c11bce03a6d51c24fc9a2d8ac872aaf41a20af5045" +} diff --git a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json index 99269c9851..54e94cfb8f 100644 --- a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json +++ b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json @@ -18,8 +18,8 @@ "Left": [] }, "nullable": [ - true, - false + false, + true ] }, "hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76" diff --git a/backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json b/backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json deleted file mode 100644 index a49baeefaf..0000000000 --- a/backend/.sqlx/query-b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_runtime (id) SELECT unnest($1::uuid[])", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray" - ] - }, - "nullable": [] - }, - "hash": "b4a9abcb38997587b28655b0f4a212a5bd4039b57fab20b163617e33a4c9dd46" -} diff --git a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json b/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json deleted file mode 100644 index 0e42bd0fdb..0000000000 --- a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9 FROM generate_series(1, $10)) RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Int8", - "Varchar", - { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlescriptflow", - "flowscript", - "flownode", - "appscript", - "aiagent" - ] - } - } - }, - { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java", - "duckdb", - "ruby" - ] - } - } - }, - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238" -} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 593de8a8e5..85b8eb06f0 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -219,9 +219,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.99" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "arbitrary" @@ -1923,9 +1923,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.37" +version = "1.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65193589c6404eb80b450d618eaf9a2cafaaafd57ecce47370519ef674a7bd44" +checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" dependencies = [ "find-msvc-tools", "jobserver", @@ -2032,9 +2032,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.47" +version = "4.5.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" +checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" dependencies = [ "clap_builder", "clap_derive", @@ -2042,9 +2042,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.47" +version = "4.5.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" +checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" dependencies = [ "anstream", "anstyle", @@ -5252,9 +5252,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" [[package]] name = "fixedbitset" @@ -11770,9 +11770,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.14.0" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" dependencies = [ "base64 0.22.1", "chrono", @@ -11790,11 +11790,11 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.14.0" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" dependencies = [ - "darling 0.20.11", + "darling 0.21.3", "proc-macro2", "quote", "syn 2.0.106", @@ -13480,11 +13480,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.43" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", "serde", @@ -13893,9 +13894,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" +checksum = "4cf893c33be71572e0e9aa6dd15e6677937abd686b066eac3f8cd3531688a627" dependencies = [ "winnow 0.7.13", ] @@ -15128,7 +15129,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "aws-sdk-config", @@ -15188,7 +15189,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "argon2", @@ -15308,7 +15309,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.544.2" +version = "1.545.0" dependencies = [ "base64 0.22.1", "chrono", @@ -15323,7 +15324,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.544.2" +version = "1.545.0" dependencies = [ "chrono", "serde", @@ -15336,7 +15337,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "axum", @@ -15355,7 +15356,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "async-recursion", @@ -15439,7 +15440,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.544.2" +version = "1.545.0" dependencies = [ "regex", "serde", @@ -15454,7 +15455,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "bytes", @@ -15478,7 +15479,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.544.2" +version = "1.545.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15490,7 +15491,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.544.2" +version = "1.545.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15499,7 +15500,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "lazy_static", @@ -15511,7 +15512,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "serde_json", @@ -15523,7 +15524,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "gosyn", @@ -15535,7 +15536,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "lazy_static", @@ -15547,7 +15548,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "serde_json", @@ -15559,7 +15560,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "nu-parser", @@ -15570,7 +15571,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15581,7 +15582,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15593,7 +15594,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "async-recursion", @@ -15616,7 +15617,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "lazy_static", @@ -15630,7 +15631,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15647,7 +15648,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "lazy_static", @@ -15661,7 +15662,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "lazy_static", @@ -15679,7 +15680,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -15704,7 +15705,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "serde_json", @@ -15714,7 +15715,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "async-recursion", @@ -15747,7 +15748,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.544.2" +version = "1.545.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15757,7 +15758,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.544.2" +version = "1.545.0" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ac9af8e0e1..8b6d9b4b60 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.544.2" +version = "1.545.0" authors.workspace = true edition.workspace = true @@ -34,7 +34,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.544.2" +version = "1.545.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -102,6 +102,10 @@ ruby = ["windmill-worker/ruby"] all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java", "ruby"] # For windows we have another set of languages enabled all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] +all_sqlx_features = ["all_languages", "enterprise", "enterprise_saml", "loki", "embedding", "parquet", "prometheus", "flow_testing", + "openidconnect", "cloud", "jemalloc", "tantivy", "sqlx", "kafka", "nats", "otel", "dind", "websocket", "http_trigger", + "postgres_trigger", "mcp", "mqtt_trigger", "sqs_trigger", "gcp_trigger", "smtp", "stripe", + "license", "oauth2", "zip", "static_frontend", "scoped_cache", "agent_worker_server"] [patch.crates-io] object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 694b79ccac..b6e5157568 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -1dc77acb47053b2dd84569d70884456de0188290 +aa4c7936b99d0de2f7af31cde79dfc7b1b482c29 \ No newline at end of file diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 3b181f1078..bc63d2f8af 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -363,11 +363,11 @@ async fn parse_python_imports_inner( // This way we make sure there is no multiple annotations for same script // and we get detailed span on conflicting versions - #[derive(serde::Serialize, serde::Deserialize)] - struct InlineMetadata { - requires_python: String, - dependencies: Vec, - } + // #[derive(serde::Serialize, serde::Deserialize)] + // struct InlineMetadata { + // requires_python: String, + // dependencies: Vec, + // } let find_requirements = code.lines().find_position(|x| { x.starts_with("#requirements:") diff --git a/backend/src/main.rs b/backend/src/main.rs index 87a148d5c3..9b39094236 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -300,6 +300,19 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { } async fn windmill_main() -> anyhow::Result<()> { + let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); + let mut monitor_killpill_rx = killpill_tx.subscribe(); + let (killpill_phase2_tx, _killpill_phase2_rx) = tokio::sync::broadcast::channel::<()>(2); + let server_killpill_rx = killpill_phase2_tx.subscribe(); + + let shutdown_tx = killpill_tx.clone(); + let shutdown_rx = killpill_tx.subscribe(); + tokio::spawn(async move { + if let Err(e) = windmill_common::shutdown_signal(shutdown_tx, shutdown_rx).await { + tracing::error!("Error in shutdown signal: {e:#}"); + } + }); + dotenv::dotenv().ok(); update_ca_certificates_if_requested(); @@ -463,13 +476,18 @@ async fn windmill_main() -> anyhow::Result<()> { if !skip_migration { // migration code to avoid break - migration_handle = windmill_api::migrate_db(&db).await?; + migration_handle = windmill_api::migrate_db(&db, killpill_rx.resubscribe()).await?; } else { tracing::info!("SKIP_MIGRATION set, skipping db migration...") } } } + if killpill_rx.try_recv().is_ok() { + tracing::info!("Received early killpill, aborting startup"); + return Ok(()); + } + let worker_mode = num_workers > 0; let conn = if mode == Mode::Agent { @@ -483,14 +501,6 @@ async fn windmill_main() -> anyhow::Result<()> { Connection::Sql(db) }; - let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); - let mut monitor_killpill_rx = killpill_tx.subscribe(); - let (killpill_phase2_tx, _killpill_phase2_rx) = tokio::sync::broadcast::channel::<()>(2); - let server_killpill_rx = killpill_phase2_tx.subscribe(); - - let shutdown_signal = - windmill_common::shutdown_signal(killpill_tx.clone(), killpill_tx.subscribe()); - #[cfg(feature = "enterprise")] tracing::info!( " @@ -1257,10 +1267,9 @@ Windmill Community Edition {GIT_VERSION} } if mcp_mode { - futures::try_join!(shutdown_signal, workers_f, server_f)?; + futures::try_join!(workers_f, server_f)?; } else { futures::try_join!( - shutdown_signal, workers_f, monitor_f, server_f, @@ -1285,6 +1294,7 @@ Windmill Community Edition {GIT_VERSION} } } } + std::process::exit(0); Ok(()) } diff --git a/backend/update_sqlx.sh b/backend/update_sqlx.sh index d8bc473f29..77b772c970 100755 --- a/backend/update_sqlx.sh +++ b/backend/update_sqlx.sh @@ -23,7 +23,7 @@ if [[ "$(uname)" == "Darwin" ]]; then sed -i '' 's/^# \(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/\1/' Cargo.toml fi -cargo sqlx prepare --workspace -- --all-targets --all-features +cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features,private # Undo the samael changes on macOS if [[ "$(uname)" == "Darwin" ]]; then diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c0e3cf4000..36b888f400 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.544.2 + version: 1.545.0 title: Windmill API contact: @@ -7727,6 +7727,10 @@ paths: - job parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: force_cancel + in: query + schema: + type: boolean requestBody: description: uuids of the jobs to cancel required: true @@ -18318,17 +18322,6 @@ components: - script_path - git_repo_resource_path - UploadFilePart: - type: object - properties: - part_number: - type: integer - tag: - type: string - required: - - part_number - - tag - MetricMetadata: type: object properties: diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 849ddf877c..8254e544e7 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -147,14 +147,14 @@ fn is_false(b: &bool) -> bool { !b } -#[derive(FromRow, Serialize, Deserialize)] -pub struct AppVersion { - pub id: i64, - pub app_id: Uuid, - pub value: sqlx::types::Json>, - pub created_by: String, - pub created_at: chrono::DateTime, -} +// #[derive(FromRow, Serialize, Deserialize)] +// pub struct AppVersion { +// pub id: i64, +// pub app_id: Uuid, +// pub value: sqlx::types::Json>, +// pub created_by: String, +// pub created_at: chrono::DateTime, +// } #[derive(Debug, Serialize, Deserialize, FromRow)] pub struct AppWithLastVersion { @@ -180,15 +180,6 @@ pub struct AppWithLastVersionAndStarred { pub starred: Option, } -#[cfg(feature = "enterprise")] -#[derive(Debug, Serialize, FromRow)] -pub struct AppWithLastVersionAndWorkspace { - #[sqlx(flatten)] - #[serde(flatten)] - pub app: AppWithLastVersion, - pub workspace_id: String, -} - #[derive(Serialize, Deserialize, FromRow)] pub struct AppWithLastVersionAndDraft { #[sqlx(flatten)] diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index e5d8c433f1..c7ab6feeec 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -101,14 +101,20 @@ impl Migrate for CustomMigrator { let mut r = false; while !r { - r = sqlx::query_scalar!("SELECT pg_try_advisory_lock($1)", lock_id) - .fetch_one(&mut *self.inner) + r = match tokio::time::timeout(std::time::Duration::from_secs(5), sqlx::query_scalar!("SELECT pg_try_advisory_lock($1)", lock_id) + .fetch_one(&mut *self.inner)) .await - .map_err(|e| { - tracing::error!("Error acquiring lock: {e:#}"); - sqlx::migrate::MigrateError::Execute(e) - })? - .unwrap_or(false); + { + Ok(Ok(r)) => r.unwrap_or(false), + Ok(Err(e)) => { + tracing::error!("Error acquiring lock: {e:#}"); + return Err(sqlx::migrate::MigrateError::Execute(e)); + } + Err(e) => { + tracing::error!("Timed out acquiring lock retrying in 5s: {e:#}"); + false + } + }; if !r { tracing::info!("PG migration lock already acquired by another server or worker, a migration is in progress, this may take a long time if you have many jobs and be normal, rechecking in 5s."); tokio::time::sleep(std::time::Duration::from_secs(5)).await; @@ -196,7 +202,10 @@ impl Migrate for CustomMigrator { } } -pub async fn migrate(db: &DB) -> Result>, Error> { +pub async fn migrate( + db: &DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> Result>, Error> { let migrator = db.acquire().await?; let mut custom_migrator = CustomMigrator { inner: migrator }; @@ -211,20 +220,27 @@ pub async fn migrate(db: &DB) -> Result>, Error> { tracing::info!("Could not remove sqlx migrations: {err:#}"); } - match sqlx::migrate!("../migrations") - .run_direct(&mut custom_migrator) - .await - { - Ok(_) => Ok(()), - Err(sqlx::migrate::MigrateError::VersionMissing(e)) => { - tracing::error!("Database had been applied more migrations than this container. - This usually mean than another container on a more recent version migrated the database and this one is on an earlier version. - Please update the container to latest. Not critical, but may cause issues if migration introduced a breaking change. Version missing: {e:#}"); - custom_migrator.unlock().await?; - Ok(()) + tokio::select! { + _ = killpill_rx.recv() => { + tracing::info!("Killpill received, stopping migration"); + return Ok(None); } - Err(err) => Err(err), - }?; + migration_result = sqlx::migrate!("../migrations") + .run_direct(&mut custom_migrator) + => { + match migration_result { + Ok(_) => Ok(()), + Err(sqlx::migrate::MigrateError::VersionMissing(e)) => { + tracing::error!("Database had been applied more migrations than this container. + This usually mean than another container on a more recent version migrated the database and this one is on an earlier version. + Please update the container to latest. Not critical, but may cause issues if migration introduced a breaking change. Version missing: {e:#}"); + custom_migrator.unlock().await?; + Ok(()) + } + Err(err) => Err(err), + }?; + } + } return crate::live_migrations::custom_migrations(&mut custom_migrator, db).await; } diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 90e63f0b76..defb53db76 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -30,15 +30,6 @@ use axum::response::Response; #[cfg(all(feature = "parquet", not(feature = "private")))] use serde::Deserialize; -#[derive(Deserialize)] -#[cfg(not(feature = "private"))] -pub struct LoadImagePreviewQuery { - #[allow(dead_code)] - pub file_key: String, - #[allow(dead_code)] - pub storage: Option, -} - #[derive(Deserialize)] #[cfg(not(feature = "private"))] pub struct DownloadFileQuery { diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 1b75070217..d1815a4994 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -1751,6 +1751,7 @@ pub struct RunJobQuery { pub timeout: Option, pub cache_ttl: Option, pub skip_preprocessor: Option, + pub poll_delay_ms: Option, } impl RunJobQuery { @@ -2055,6 +2056,7 @@ async fn cancel_jobs( db: &DB, username: &str, w_id: &str, + force_cancel: bool, ) -> error::JsonResult> { let mut uuids = vec![]; let mut tx = db.begin().await?; @@ -2114,7 +2116,7 @@ async fn cancel_jobs( w_id, tx, db, - false, + force_cancel, false, ) .await?; @@ -2145,11 +2147,17 @@ async fn cancel_jobs( Ok(Json(uuids)) } +#[derive(Deserialize)] +pub struct CancelSelectionQuery { + force_cancel: Option, +} + async fn cancel_selection( authed: ApiAuthed, Extension(db): Extension, Extension(user_db): Extension, Path(w_id): Path, + Query(query): Query, Json(jobs): Json>, ) -> error::JsonResult> { let mut tx = user_db.begin(&authed).await?; @@ -2163,7 +2171,14 @@ async fn cancel_selection( .await?; tx.commit().await?; - cancel_jobs(jobs_to_cancel, &db, authed.username.as_str(), w_id.as_str()).await + cancel_jobs( + jobs_to_cancel, + &db, + authed.username.as_str(), + w_id.as_str(), + query.force_cancel.unwrap_or(false), + ) + .await } async fn list_filtered_job_uuids( @@ -5280,6 +5295,7 @@ pub async fn stream_job( .to_args_from_runnable(&db, &w_id, runnable_id.clone(), run_query.skip_preprocessor) .await?; + let poll_delay_ms = run_query.poll_delay_ms; let uuid = match runnable_id { RunnableId::ScriptId(ScriptId::ScriptPath(script_path)) | RunnableId::HubScript(script_path) => { @@ -5349,6 +5365,7 @@ pub async fn stream_job( None, None, tx, + poll_delay_ms, ); let body = axum::body::Body::from_stream(stream.map(Result::<_, std::convert::Infallible>::Ok)); @@ -6521,6 +6538,7 @@ pub struct JobUpdateQuery { pub only_result: Option, pub fast: Option, pub is_flow: Option, + pub poll_delay_ms: Option, } #[derive(Serialize, Debug)] @@ -6682,6 +6700,7 @@ async fn get_job_update_sse( only_result, fast, is_flow, + poll_delay_ms, }): Query, ) -> error::Result { let (tx, rx) = tokio::sync::mpsc::channel(32); @@ -6701,6 +6720,7 @@ async fn get_job_update_sse( no_logs, is_flow, tx, + poll_delay_ms, ); let stream = tokio_stream::wrappers::ReceiverStream::new(rx).map(|x| { @@ -6746,6 +6766,7 @@ fn start_job_update_sse_stream( no_logs: Option, is_flow: Option, tx: tokio::sync::mpsc::Sender, + poll_delay_ms: Option, ) -> () { tokio::spawn(async move { let mut log_offset = initial_log_offset; @@ -6828,13 +6849,27 @@ fn start_job_update_sse_stream( loop { i += 1; - let ms_duration = if i > 100 || !fast.unwrap_or(false) { + let mut ms_duration = if i > 100 || !fast.unwrap_or(false) { 3000 } else if i > 10 { 500 } else { 100 }; + + if let Some(poll_delay_ms) = poll_delay_ms { + #[cfg(feature = "enterprise")] + if poll_delay_ms < 50 { + tracing::warn!("Poll delay ms is less than 50, setting it to 50"); + ms_duration = 50; + } else { + ms_duration = poll_delay_ms; + } + + #[cfg(not(feature = "enterprise"))] + tracing::warn!("Settable poll delay requires EE"); + } + if last_ping.elapsed().as_secs() > 5 { if tx.send(JobUpdateSSEStream::Ping).await.is_err() { tracing::warn!("Failed to send job ping for job {job_id}"); diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index eec6574131..3f8f3a993a 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -100,10 +100,10 @@ mod inkeep_oss; mod inputs; mod integration; mod live_migrations; +pub mod openapi; #[cfg(all(feature = "private", feature = "parquet"))] pub mod s3_proxy_ee; mod s3_proxy_oss; -pub mod openapi; mod approvals; #[cfg(all(feature = "enterprise", feature = "private"))] @@ -155,10 +155,10 @@ pub mod stripe_ee; #[cfg(all(feature = "stripe", feature = "enterprise"))] mod stripe_oss; #[cfg(feature = "private")] -pub mod teams_ee; -#[cfg(feature = "private")] pub mod teams_cache_ee; mod teams_cache_oss; +#[cfg(feature = "private")] +pub mod teams_ee; mod teams_oss; mod token; mod tracing_init; @@ -696,9 +696,10 @@ pub async fn run_server( name.map(|x| format!("name={x}")).unwrap_or_default() ); - port_tx - .send(format!("http://localhost:{}", port)) - .expect("Failed to send port"); + if let Err(e) = port_tx.send(format!("http://localhost:{}", port)) { + tracing::error!("Failed to send port: {e:#}"); + return Err(anyhow::anyhow!("Failed to send port, exiting early: {e:#}")); + } let server = server.with_graceful_shutdown(async move { killpill_rx.recv().await.ok(); @@ -794,8 +795,11 @@ async fn openapi_json() -> Response { .unwrap() } -pub async fn migrate_db(db: &DB) -> anyhow::Result>> { - db::migrate(db) +pub async fn migrate_db( + db: &DB, + killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> anyhow::Result>> { + db::migrate(db, killpill_rx) .await .map_err(|e| anyhow::anyhow!("Error migrating db: {e:#}")) } diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index 1165a5e2f3..4e6c5ab086 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -71,10 +71,10 @@ struct WorkerPing { wm_memory_usage: Option, } -#[derive(Serialize, Deserialize)] -struct EnableWorkerQuery { - disable: bool, -} +// #[derive(Serialize, Deserialize)] +// struct EnableWorkerQuery { +// disable: bool, +// } #[derive(Deserialize)] pub struct ListWorkerQuery { diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 1b9d928cf4..711de8f68d 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -261,11 +261,11 @@ pub struct WorkspaceSettings { pub auto_add_instance_groups_roles: Option, } -#[derive(sqlx::Type, Serialize, Deserialize, Debug)] -#[sqlx(type_name = "WORKSPACE_KEY_KIND", rename_all = "lowercase")] -pub enum WorkspaceKeyKind { - Cloud, -} +// #[derive(sqlx::Type, Serialize, Deserialize, Debug)] +// #[sqlx(type_name = "WORKSPACE_KEY_KIND", rename_all = "lowercase")] +// pub enum WorkspaceKeyKind { +// Cloud, +// } #[derive(Deserialize)] struct EditCommandScript { @@ -1488,9 +1488,9 @@ async fn delete_git_sync_repository( )) } +#[cfg(feature = "enterprise")] #[derive(Debug, Deserialize)] struct EditDeployUIConfig { - #[cfg(feature = "enterprise")] deploy_ui_settings: Option, } @@ -2553,7 +2553,7 @@ async fn clone_variables( source_workspace_id: &str, target_workspace_id: &str, ) -> Result<()> { - sqlx::query!( + sqlx::query!( "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at) SELECT $2, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at FROM variable diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 98dac87744..af34bc3513 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -18,7 +18,7 @@ use std::{ }, }; -use tokio::sync::broadcast; +use tokio::{spawn, sync::broadcast}; use ee_oss::CriticalErrorChannel; use error::Error; @@ -198,8 +198,47 @@ pub async fn shutdown_signal( }, } + spawn(async move { + #[cfg(any(target_os = "linux", target_os = "macos"))] + tokio::select! { + _ = terminate() => { + tracing::info!("2nd shutdown monitor received terminate"); + }, + _ = tokio::signal::ctrl_c() => { + tracing::info!("2nd shutdown monitor received ctrl-c"); + }, + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + tokio::select! { + _ = tokio::signal::ctrl_c() => {}, + _ = rx.recv() => { + tracing::info!("2nd shutdown monitor received killpill"); + }, + } + + tracing::info!("Second terminate signal received, forcefully exiting"); + + let handle = tokio::runtime::Handle::current(); + let metrics = handle.metrics(); + tracing::info!( + "Alive tasks: {}, global queue depth: {}", + metrics.num_alive_tasks(), + metrics.global_queue_depth() + ); + + std::process::exit(1); + }); + tracing::info!("signal received, starting graceful shutdown"); let _ = tx.send(); + + spawn(async move { + tokio::time::sleep(std::time::Duration::from_hours(24 * 7)).await; + tracing::info!("Forcefully exiting after 7 days"); + std::process::exit(1); + }); + Ok(()) } diff --git a/backend/windmill-worker/loader.py b/backend/windmill-worker/loader.py index 1361e06ead..c387e5e0fc 100644 --- a/backend/windmill-worker/loader.py +++ b/backend/windmill-worker/loader.py @@ -51,25 +51,32 @@ class WindmillFinder(MetaPathFinder): url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py{query_params}" req = urllib.request.Request(url, None, headers) - try: - req_start = time.time() - with urllib.request.urlopen(req) as response: - os.makedirs(folder, exist_ok=True) - r = response.read().decode("utf-8") - if r == "WINDMILL_IS_FOLDER": - return ModuleSpec(name, WindmillLoader(name)) - with open(fullpath, "w+") as f: - f.write(r) - return ModuleSpec(name, SourceFileLoader(name, fullpath)) - except urllib.error.HTTPError as e: - duration = time.time() - req_start - if e.code != 404: - print(f"Error fetching script {script_path}: HTTP {e.code} - {e.reason} - {duration}s") - return ModuleSpec(name, WindmillLoader(name)) - except Exception as e: - duration = time.time() - req_start - print(f"Error fetching script {script_path}: {e} - {duration}s") - return ModuleSpec(name, WindmillLoader(name)) + + for attempt in range(4): # 0, 1, 2, 3 = up to 3 retries + try: + req_start = time.time() + with urllib.request.urlopen(req) as response: + os.makedirs(folder, exist_ok=True) + r = response.read().decode("utf-8") + if r == "WINDMILL_IS_FOLDER": + return ModuleSpec(name, WindmillLoader(name)) + with open(fullpath, "w+") as f: + f.write(r) + return ModuleSpec(name, SourceFileLoader(name, fullpath)) + except urllib.error.HTTPError as e: + duration = time.time() - req_start + if e.code != 404: + print(f"Error fetching script {script_path}: HTTP {e.code} - {e.reason} - {duration}s") + return ModuleSpec(name, WindmillLoader(name)) + except Exception as e: + duration = time.time() - req_start + # Check if this is errno 104 (Connection reset by peer) and we have retries left + if (hasattr(e, 'errno') and e.errno == 104) and attempt < 3: + print(f"Connection reset (errno 104) fetching script {script_path}, retrying in 3s (attempt {attempt + 1}/3)") + time.sleep(3) + continue + print(f"Error fetching script {script_path}: {e} - {duration}s") + return ModuleSpec(name, WindmillLoader(name)) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 2ae1a42f74..06a295f820 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -791,6 +791,7 @@ pub fn start_interactive_worker_shell( loop { if let Ok(_) = killpill_rx.try_recv() { + tracing::info!("Received killpill, exiting worker shell"); break; } else { let pulled_job = match &conn { @@ -2041,8 +2042,14 @@ pub async fn run_worker( } tracing::info!(worker = %worker_name, hostname = %hostname, "waiting for interactive_shell to finish"); if let Some(interactive_shell) = interactive_shell { - if let Err(e) = interactive_shell.await { - tracing::error!("error in awaiting interactive_shell process: {e:?}") + match tokio::time::timeout(Duration::from_secs(10), interactive_shell).await { + Ok(Ok(_)) => {} + Ok(Err(e)) => { + tracing::error!("error in interactive_shell process: {e:?}") + } + Err(_) => { + tracing::error!("timed out awaiting interactive_shell process") + } } } tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name); diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 832af368d1..afd2b6e850 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -526,38 +526,25 @@ pub async fn update_flow_status_after_job_completion_internal( } else { None }; - tracing::error!( - "flow_job_duration: {:?}, position: {:?} flow: {:?}", - flow_job_duration, - position, - flow - ); + let nindex = if let Some(position) = position { sqlx::query_scalar!( - "UPDATE v2_job_status SET flow_status = - CASE - WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN - JSONB_SET( - JSONB_SET(JSONB_SET(JSONB_SET( + " + UPDATE v2_job_status SET flow_status = + JSONB_SET(JSONB_SET(JSONB_SET(JSONB_SET( flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), - ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), - ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6), - ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb - ) - ELSE - JSONB_SET(JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4), - ARRAY['modules', $1::TEXT, 'iterator', 'index'], - ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb - ) END + ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb), + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'started_at', $3::TEXT], $5), + ARRAY['modules', $1::TEXT, 'flow_jobs_duration', 'duration_ms', $3::TEXT], $6) WHERE id = $2 RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int", old_status.step, flow, position as i32, json!(success), - flow_job_duration.as_ref().map(|x| json!(x.started_at)), - flow_job_duration.as_ref().map(|x| json!(x.duration_ms)) + flow_job_duration.as_ref().map(|x| json!(x.started_at)).unwrap_or_default(), + flow_job_duration.as_ref().map(|x| json!(x.duration_ms)).unwrap_or_default(), ) } else { sqlx::query_scalar!( @@ -578,8 +565,7 @@ pub async fn update_flow_status_after_job_completion_internal( Error::internal_err(format!( "error while fetching iterator index: {e:#}" )) - })? - .ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress")))?; + })?.ok_or_else(|| Error::internal_err(format!("requiring an index in InProgress for flow {flow} at step {}", old_status.step)))?; // let status_for_debug = sqlx::query!( // "SELECT flow_status FROM v2_job_status WHERE id = $1", @@ -611,7 +597,7 @@ pub async fn update_flow_status_after_job_completion_internal( sqlx::query_scalar!( "UPDATE v2_job_status SET flow_status = CASE - WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN + WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN JSONB_SET( JSONB_SET(JSONB_SET(JSONB_SET( flow_status, @@ -634,8 +620,8 @@ pub async fn update_flow_status_after_job_completion_internal( flow, position as i32, json!(success), - flow_job_duration.as_ref().map(|x| json!(x.started_at)), - flow_job_duration.as_ref().map(|x| json!(x.duration_ms)) + flow_job_duration.as_ref().map(|x| json!(x.started_at)).unwrap_or_default(), + flow_job_duration.as_ref().map(|x| json!(x.duration_ms)).unwrap_or_default(), ) } else { sqlx::query_scalar!( @@ -1599,7 +1585,7 @@ async fn set_success_and_duration_in_flow_job_success<'c>( if let Some(position) = position { sqlx::query!( "UPDATE v2_job_status SET flow_status = - CASE WHEN flow_status->'modules'->$1::TEXT->'flow_jobs_duration' IS NOT NULL THEN + CASE WHEN flow_status->'modules'->$1::int->'flow_jobs_duration' IS NOT NULL THEN JSONB_SET(JSONB_SET(JSONB_SET( flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], @@ -1615,8 +1601,8 @@ async fn set_success_and_duration_in_flow_job_success<'c>( flow, position as i32, json!(success), - flow_job_duration.as_ref().map(|x| json!(x.duration_ms)), - flow_job_duration.as_ref().map(|x| json!(x.started_at)) + flow_job_duration.as_ref().map(|x| json!(x.duration_ms)).unwrap_or_default(), + flow_job_duration.as_ref().map(|x| json!(x.started_at)).unwrap_or_default() ) .execute(&mut **tx) .await diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index d654e5af65..366cc9cdf7 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -308,7 +308,7 @@ pub(crate) async fn queue_vacuum(conn: &Connection, worker_name: &str, hostname: tokio::task::spawn( (async move { tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue"); - if let Err(e) = sqlx::query!("VACUUM v2_job_queue, v2_job_runtime, v2_job_status, job_perms") + if let Err(e) = sqlx::query!("VACUUM (SKIP_LOCKED) v2_job_queue, v2_job_runtime, v2_job_status, job_perms") .execute(&db2) .await { diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 4a6a1864f9..4371d3aacf 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.544.2"; +export const VERSION = "v1.545.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 5db4b1765d..034fdbeb1a 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.544.2"; +export const VERSION = "1.545.0"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/docker/DockerfileFull b/docker/DockerfileFull index 1d5f639113..0cf1a83de2 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -1,8 +1,8 @@ FROM ghcr.io/windmill-labs/windmill:dev # Rust -COPY --from=rust:1.88.0 /usr/local/cargo /usr/local/cargo -COPY --from=rust:1.88.0 /usr/local/rustup /usr/local/rustup +COPY --from=rust:1.90.0 /usr/local/cargo /usr/local/cargo +COPY --from=rust:1.90.0 /usr/local/rustup /usr/local/rustup RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 376f994f89..3952113c01 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -20,8 +20,8 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ FROM ghcr.io/windmill-labs/windmill-ee:dev # Rust -COPY --from=rust:1.88.0 /usr/local/cargo /usr/local/cargo -COPY --from=rust:1.88.0 /usr/local/rustup /usr/local/rustup +COPY --from=rust:1.90.0 /usr/local/cargo /usr/local/cargo +COPY --from=rust:1.90.0 /usr/local/rustup /usr/local/rustup RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7a7d8af914..648f12a6c8 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.544.2", + "version": "1.545.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.544.2", + "version": "1.545.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index a214dd13cf..3946274a02 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.544.2", + "version": "1.545.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 8ba6ef3b36..4e7b4f8c81 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -541,7 +541,7 @@
{/key} - {#if !disabled && itemPicker && extra?.['disableVariablePicker'] != true} - - - {/if} + {/if} + {#if !disabled && itemPicker && extra?.['disableVariablePicker'] != true} + + {/if}
{@render variableInput()} diff --git a/frontend/src/lib/components/FlowLoopIterationPreview.svelte b/frontend/src/lib/components/FlowLoopIterationPreview.svelte index c4b0a5080c..6c296d0804 100644 --- a/frontend/src/lib/components/FlowLoopIterationPreview.svelte +++ b/frontend/src/lib/components/FlowLoopIterationPreview.svelte @@ -176,7 +176,7 @@
{#if jobId} { job = newJob diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 558a854bca..f2340066e9 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -583,7 +583,7 @@ bind:suspendStatus hideDownloadInGraph={customUi?.downloadLogs === false} wideResults - bind:flowStateStore={flowStateStore.val} + bind:flowState={flowStateStore.val} {jobId} onDone={() => { isRunning = false diff --git a/frontend/src/lib/components/FlowStatusViewer.svelte b/frontend/src/lib/components/FlowStatusViewer.svelte index 6ca2c8fd0f..7d5c1c4a12 100644 --- a/frontend/src/lib/components/FlowStatusViewer.svelte +++ b/frontend/src/lib/components/FlowStatusViewer.svelte @@ -11,7 +11,7 @@ jobId: string initialJob?: Job | undefined workspaceId?: string | undefined - flowStateStore?: FlowState + flowState?: FlowState selectedJobStep?: string | undefined hideFlowResult?: boolean hideTimeline?: boolean @@ -39,7 +39,7 @@ jobId, initialJob = undefined, workspaceId = undefined, - flowStateStore = $bindable({}), + flowState = $bindable({}), selectedJobStep = $bindable(undefined), hideFlowResult = false, hideTimeline = false, @@ -67,7 +67,7 @@ let globalRefreshes: Record Promise)[]> = $state({}) setContext('FlowStatusViewer', { - flowStateStore, + flowState: flowState, suspendStatus, retryStatus, hideDownloadInGraph, @@ -88,7 +88,7 @@ retryStatus.val = {} suspendStatus.val = {} globalRefreshes = {} - flowStateStore.val = {} + flowState.val = {} localDurationStatuses = {} localModuleStates = {} } diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index caf8c7083b..dc101fa520 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -48,7 +48,7 @@ import JobAssetsViewer from './assets/JobAssetsViewer.svelte' let { - flowStateStore, + flowState: flowStateStore, retryStatus, suspendStatus, hideDownloadInGraph, diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index a266dd1f89..ddd5329a04 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -61,7 +61,6 @@ attempted_at: string } | null = $state(null) - function showSetting(setting: string, values: Record) { if (setting == 'dev_instance') { if (values['license_key'] == undefined) { @@ -160,18 +159,19 @@ fetch_available_python_versions() } - function handleTeamChange( teamItem: { team_id: string; team_name: string } | undefined, i: number ) { $values['critical_error_channels'][i] = { - teams_channel: teamItem ? { - team_id: teamItem.team_id, - team_name: teamItem.team_name, - channel_id: undefined, // Will be set when channel is selected - channel_name: undefined - } : undefined + teams_channel: teamItem + ? { + team_id: teamItem.team_id, + team_name: teamItem.team_name, + channel_id: undefined, // Will be set when channel is selected + channel_name: undefined + } + : undefined } } @@ -191,7 +191,6 @@ } } } - @@ -367,7 +366,7 @@ { + onKeyDown={() => { licenseKeyChanged = true }} bind:password={$values[setting.key]} @@ -600,7 +599,8 @@ : undefined, (channel) => handleChannelChange(channel, i) } - onError={(e) => sendUserToast('Failed to load channels: ' + e.message, true)} + onError={(e) => + sendUserToast('Failed to load channels: ' + e.message, true)} /> {/if}
@@ -645,8 +645,8 @@ if ($values[setting.key] == undefined || !Array.isArray($values[setting.key])) { $values[setting.key] = [] } - // Start with a typed default to avoid invalid primitives in the array - $values[setting.key] = $values[setting.key].concat({ email: '' }) + // Start with a typed default to avoid invalid primitives in the array + $values[setting.key] = $values[setting.key].concat({ email: '' }) }} id="arg-input-add-item" startIcon={{ icon: Plus }} diff --git a/frontend/src/lib/components/Password.svelte b/frontend/src/lib/components/Password.svelte index 2d031da318..29df4fa4cb 100644 --- a/frontend/src/lib/components/Password.svelte +++ b/frontend/src/lib/components/Password.svelte @@ -1,15 +1,28 @@ @@ -29,7 +42,7 @@ : ''} text-sm h-9" type="password" bind:value={password} - on:keydown + onkeydown={onKeyDown} autocomplete="new-password" {placeholder} {disabled} @@ -41,7 +54,7 @@ : ''} text-sm h-9" type="text" bind:value={password} - on:keydown + onkeydown={bubble('keydown')} autocomplete="new-password" {placeholder} {disabled} diff --git a/frontend/src/lib/components/PasswordArgInput.svelte b/frontend/src/lib/components/PasswordArgInput.svelte index e4b349b531..200d15d49a 100644 --- a/frontend/src/lib/components/PasswordArgInput.svelte +++ b/frontend/src/lib/components/PasswordArgInput.svelte @@ -2,24 +2,32 @@ import { VariableService } from '$lib/gen' import { userStore, workspaceStore } from '$lib/stores' import { generateRandomString } from '$lib/utils' + import { Button } from './common' import Password from './Password.svelte' + import { untrack } from 'svelte' - export let value: string | undefined = undefined - export let disabled: boolean + interface Props { + value?: string | undefined + disabled: boolean + } - let path = '' - let password = value && typeof value === 'string' && !value.startsWith('$var:') ? value : '' + let { value = $bindable(undefined), disabled }: Props = $props() + + let path = $state('') + let password = $state( + value && typeof value === 'string' && !value.startsWith('$var:') ? value : '' + ) let isGenerating = false + + let userPrefix = $derived( + 'u/' + ($userStore?.username ?? $userStore?.email)?.split('@')[0] + '/secret_arg/' + ) async function generateValue() { if (isGenerating) return isGenerating = true try { - let npath = - 'u/' + - ($userStore?.username ?? $userStore?.email)?.split('@')[0] + - '/secret_arg/' + - generateRandomString(12) + let npath = userPrefix + generateRandomString(12) let nvalue = '$var:' + npath await VariableService.createVariable({ workspace: $workspaceStore!, @@ -32,6 +40,7 @@ } }) path = npath + console.log('generated', nvalue) value = nvalue debouncedUpdate() } finally { @@ -59,13 +68,33 @@ timeout = setTimeout(updateValue, 500) } - $: password && debouncedUpdate() + $effect(() => { + password && untrack(() => debouncedUpdate()) + }) - $: $workspaceStore && - ($userStore?.username || $userStore?.email) && - path == '' && - password != '' && - generateValue() + $effect(() => { + $workspaceStore && + ($userStore?.username || $userStore?.email) && + path == '' && + password != '' && + untrack(() => generateValue()) + }) - +{#if value?.startsWith('$var:') && !value.startsWith('$var:' + userPrefix)} +
+ Linked to static variable + +
+{:else} + +{/if} diff --git a/frontend/src/lib/components/apps/components/display/table/utils.ts b/frontend/src/lib/components/apps/components/display/table/utils.ts index 3f84b66db5..2cfb668b83 100644 --- a/frontend/src/lib/components/apps/components/display/table/utils.ts +++ b/frontend/src/lib/components/apps/components/display/table/utils.ts @@ -27,9 +27,9 @@ export abstract class AbstractCellRenderer implements ICellRendererComp { eGui: any protected component: | { - refresh: (params: ICellRendererParams) => void - destroy: () => void - } + refresh: (params: ICellRendererParams) => void + destroy: () => void + } | undefined constructor(parentElement = 'span') { // create empty span (or other element) to place svelte component in @@ -275,7 +275,7 @@ export function validateColumnDefs(columnDefs: WindmillColumnDef[]): { } // Check if 'field' property exists and is a non-empty string - if (noField && !(colDef.children && Array.isArray(colDef.children))) { + if (noField && !(colDef.children && Array.isArray(colDef.children)) && !colDef.cellRenderer) { isValid = false errors.push( `Column at index ${index} is missing a valid 'field' property nor having any children.` diff --git a/frontend/src/lib/components/flows/flowModuleNextId.ts b/frontend/src/lib/components/flows/flowModuleNextId.ts index 1d0cd14ec6..34bed860f8 100644 --- a/frontend/src/lib/components/flows/flowModuleNextId.ts +++ b/frontend/src/lib/components/flows/flowModuleNextId.ts @@ -6,7 +6,6 @@ import { charsToNumber, numberToChars } from './idUtils' // Computes the next available id export function nextId(flowState: FlowState, fullFlow: OpenFlow): string { const allIds = dfs(fullFlow.value.modules, (fm) => fm.id) - console.log('allIds', allIds) const max = allIds.concat(Object.keys(flowState)).reduce((acc, key) => { if (key.length >= 4) { diff --git a/frontend/src/lib/components/graph/model.ts b/frontend/src/lib/components/graph/model.ts index 8ed90faeeb..990d5cac53 100644 --- a/frontend/src/lib/components/graph/model.ts +++ b/frontend/src/lib/components/graph/model.ts @@ -28,7 +28,7 @@ export type DurationStatus = { } export type FlowStatusViewerContext = { - flowStateStore?: FlowState + flowState?: FlowState retryStatus: StateStore> suspendStatus: StateStore> hideDownloadInGraph?: boolean diff --git a/frontend/src/lib/components/runs/JobsLoader.svelte b/frontend/src/lib/components/runs/JobsLoader.svelte index 90500ff5f5..920609665a 100644 --- a/frontend/src/lib/components/runs/JobsLoader.svelte +++ b/frontend/src/lib/components/runs/JobsLoader.svelte @@ -70,7 +70,7 @@ jobKindsCat = undefined, minTs = $bindable(undefined), maxTs = $bindable(undefined), - jobKinds = $bindable(''), + jobKinds = $bindable(undefined), queue_count = $bindable(undefined), suspended_count = $bindable(undefined), autoRefresh = true, diff --git a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte index 7ad0ab7d80..e3309f99e0 100644 --- a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte @@ -858,6 +858,7 @@ {:then Module} | undefined = $state(undefined) let suspended_count: Tweened | undefined = $state(undefined) - let jobKinds: string | undefined = undefined + let jobKinds: string | undefined = $state(undefined) let loading: boolean = $state(false) let paths: string[] = $state([]) let usernames: string[] = $state([]) @@ -186,7 +186,7 @@ confirmBtnText: string loading?: boolean preContent?: string - onConfirm?: () => void + onConfirm?: (forceCancel: boolean) => void type?: ConfirmationModal['$$prop_def']['type'] } = $state(undefined) @@ -544,10 +544,11 @@ } } - async function cancelJobs(uuidsToCancel: string[]) { + async function cancelJobs(uuidsToCancel: string[], forceCancel: boolean = false) { const uuids = await JobService.cancelSelection({ workspace: $workspaceStore ?? '', - requestBody: uuidsToCancel + requestBody: uuidsToCancel, + forceCancel: forceCancel }) selectedIds = [] jobsLoader?.loadJobs(minTs, maxTs, true, true) @@ -556,6 +557,7 @@ } async function onCancelFilteredJobs() { + forceCancelInPopup = false askingForConfirmation = { title: 'Confirm cancelling all jobs corresponding to the selected filters', confirmBtnText: 'Loading...', @@ -570,18 +572,19 @@ title: `Confirm cancelling all jobs corresponding to the selected filters (${jobIdsToCancel.length} jobs)`, confirmBtnText: `Cancel ${jobIdsToCancel.length} jobs that matched the filters`, preContent: selectedFiltersString, - onConfirm: () => { - cancelJobs(jobIdsToCancel) + onConfirm: (forceCancel) => { + cancelJobs(jobIdsToCancel, forceCancel) } } } async function onCancelSelectedJobs() { + forceCancelInPopup = true askingForConfirmation = { confirmBtnText: `Cancel ${selectedIds.length} jobs`, title: 'Confirm cancelling the selected jobs', - onConfirm: () => { - cancelJobs(selectedIds) + onConfirm: (forceCancel) => { + cancelJobs(selectedIds, forceCancel) } } } @@ -798,6 +801,8 @@ const smallScreenWidth = 1920 const verySmallScreenWidth = 1300 + + let forceCancelInPopup = $state(false) { const func = askingForConfirmation?.onConfirm - await func?.() + await func?.(forceCancelInPopup) askingForConfirmation = undefined }} type={askingForConfirmation?.type} @@ -853,6 +858,27 @@ > {#if askingForConfirmation?.preContent} 
{askingForConfirmation.preContent}
+ + {#if forceCancelInPopup} +
+

+ Force cancel is enabled. This is dangerous, only do this if you have no alternatives. + Instead of being gracefully cancelled, all jobs will be immediately sent to the completed + job table regardless of them being processed or not or part of running flows. You may end + up in an inconsistent state. +

+
+ {/if} {/if} diff --git a/lsp/Pipfile b/lsp/Pipfile index 4912da5b48..089eda968f 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.544.2" -wmill_pg = ">=1.544.2" +wmill = ">=1.545.0" +wmill_pg = ">=1.545.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 09feda8a6c..83ae9cd191 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.544.2 + version: 1.545.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 14ef14755a..198ab04f29 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.544.2' + ModuleVersion = '1.545.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index ac69a673cb..5ec585477a 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.544.2" +version = "1.545.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 568da0aded..19e833bbc6 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.544.2" +version = "1.545.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index a85d5ea597..e677ee8fba 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.544.2", + "version": "1.545.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index c96c949fc3..272b59aac6 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.544.2", + "version": "1.545.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 69e4d3f4f8..0166cfcbe9 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.544.2 +1.545.0