diff --git a/.github/workflows/git-commands.yaml b/.github/workflows/git-commands.yaml index b866d9b0f9..7c7a05e81e 100644 --- a/.github/workflows/git-commands.yaml +++ b/.github/workflows/git-commands.yaml @@ -40,11 +40,12 @@ jobs: with: github-token: ${{ steps.app.outputs.token }} script: | + const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: 'Starting sqlx update...' + body: `Starting sqlx update...\n\n[View workflow run](${runUrl})` }) - name: Checkout repository diff --git a/.github/workflows/weekly-pr-summary.yml b/.github/workflows/weekly-pr-summary.yml index ea0d115919..821fd86b05 100644 --- a/.github/workflows/weekly-pr-summary.yml +++ b/.github/workflows/weekly-pr-summary.yml @@ -49,14 +49,14 @@ jobs: - **Bug Fixes**: PRs with titles starting with "fix:", "bug:", or containing "fix", "resolve", "patch" - **Other**: All remaining PRs (improvements, refactors, docs, chores, etc.) - 4. **Gather Details**: For each merged PR, include: + 4. **Gather Details**: For each feature and bug fix merged PR, include: - Full PR title (NO truncation, NO links) - Author (extract login from author.login in JSON) - Brief summary: Use `gh pr view --json body` to get PR description, then extract first paragraph or key points (1-2 sentences max) 5. **Character Limit Enforcement**: - - The final summary MUST be under 6000 characters - - If the summary exceeds 6000 characters, truncate PR descriptions (NOT titles) and add at the end: "_and X more PRs_" where X is the count of omitted PRs + - The final summary MUST be under 5000 characters + - If the summary exceeds 5000 characters, truncate PR descriptions (NOT titles) and add at the end: "_and X more PRs_" where X is the count of omitted PRs 6. **Save Summary to Markdown File**: Write the summary to a file for webhook delivery: - Save the complete formatted markdown to: `summary.md` @@ -65,23 +65,19 @@ jobs: ## Output Format: ```markdown - #### 📈 Weekly overview + ### 📈 Weekly overview - **Total merged**: X - **Features**: Y - **Bug Fixes**: Z - **Other**: W - #### ✨ Features (Y) - • **[Full PR Title]** by @username - [brief impact description] - • **[Full PR Title]** by @username - [brief impact description] + ### ✨ Features (Y) + - **[Full PR Title]** by @username - [brief impact description] + - **[Full PR Title]** by @username - [brief impact description] - #### 🐛 Bug Fixes (Z) - • **[Full PR Title]** by @username - [brief impact description] - • **[Full PR Title]** by @username - [brief impact description] - - #### 🔧 Other (W) - • **[Full PR Title]** by @username - [brief impact description] - • **[Full PR Title]** by @username - [brief impact description] + ### 🐛 Bug Fixes (Z) + - **[Full PR Title]** by @username - [brief impact description] + - **[Full PR Title]** by @username - [brief impact description] _and X more PRs_ ``` @@ -89,9 +85,9 @@ jobs: ## Important Notes: - **CRITICAL**: ONLY include PRs with state "merged" from the last 7 days - **CRITICAL**: EXCLUDE all PRs with titles starting with "chore: release" or "chore(release)" - - **CRITICAL**: Total character count MUST be under 6000 characters - - Only use #### markdown headers for major sections and emoji indicators - - Use bullet points (•) for individual PR entries - more compact than paragraphs + - **CRITICAL**: Total character count MUST be under 5000 characters + - Count the number of "Other" PRs but do not include a section for them in the output + - Only use ### markdown headers for major sections and emoji indicators - NO links to PRs - NO merged date in output - NEVER truncate PR titles - show full titles @@ -100,7 +96,7 @@ jobs: - If a PR has no description, write "(No description provided)" - Extract meaningful summary from PR body - look for the first paragraph or key bullet points - Parse JSON responses carefully using `jq` or similar tools - - If summary exceeds 6000 chars, shorten PR descriptions and add "_and X more PRs_" at the end + - If summary exceeds 5000 chars, shorten PR descriptions and add "_and X more PRs_" at the end - Count PRs in each category and display in both overview and section headers ## Saving the Markdown Output: diff --git a/CHANGELOG.md b/CHANGELOG.md index 02f91aaa8a..d1b0a05c89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,151 @@ # Changelog +## [1.575.0](https://github.com/windmill-labs/windmill/compare/v1.574.3...v1.575.0) (2025-11-12) + + +### Features + +* SQL result_collection flag ([#7041](https://github.com/windmill-labs/windmill/issues/7041)) ([14e41ba](https://github.com/windmill-labs/windmill/commit/14e41babfe6321abb22081b1b3601e9009451257)) + + +### Bug Fixes + +* allow --yes in fork command ([#7122](https://github.com/windmill-labs/windmill/issues/7122)) ([1280617](https://github.com/windmill-labs/windmill/commit/1280617a283ef7bcf53c17d833f1413ffe69dbf8)) +* **backend:** scim email filter ([#7120](https://github.com/windmill-labs/windmill/issues/7120)) ([7a8fd86](https://github.com/windmill-labs/windmill/commit/7a8fd865e289522fd2d5eaaa9af471f01193a03d)) +* **flow chat:** fix chat in edit mode + cleaner code ([#7118](https://github.com/windmill-labs/windmill/issues/7118)) ([9791eb4](https://github.com/windmill-labs/windmill/commit/9791eb4568aa9548174941f1bbfb6a338c3bef00)) + +## [1.574.3](https://github.com/windmill-labs/windmill/compare/v1.574.2...v1.574.3) (2025-11-11) + + +### Bug Fixes + +* **backend:** add sslmode parameter to DuckLake PostgreSQL connections ([ea5b3e6](https://github.com/windmill-labs/windmill/commit/ea5b3e6b9269a0c631c7601313ab97c4bbdc2137)) +* use teams internal_id rather than id ([#7113](https://github.com/windmill-labs/windmill/issues/7113)) ([25c36c1](https://github.com/windmill-labs/windmill/commit/25c36c1e0a6cfc56e079ff5a18980ea67403d4f9)) + +## [1.574.2](https://github.com/windmill-labs/windmill/compare/v1.574.1...v1.574.2) (2025-11-10) + + +### Bug Fixes + +* **backend:** improve pwsh param block parsing ([#7096](https://github.com/windmill-labs/windmill/issues/7096)) ([51cba95](https://github.com/windmill-labs/windmill/commit/51cba95d3961ec637fd5060556f0e7a8a8bdbb88)) +* **backend:** improve sse parsing ([#7100](https://github.com/windmill-labs/windmill/issues/7100)) ([2e245ac](https://github.com/windmill-labs/windmill/commit/2e245accc5f641f64a2cd28fc181e6a806c407e2)) +* fix multiselect in list for apps ([9e4882c](https://github.com/windmill-labs/windmill/commit/9e4882c0a919303480f5a431db3c05b1d5855d37)) +* make ai chat works with unicode messages ([2d54dfb](https://github.com/windmill-labs/windmill/commit/2d54dfbf05f21faa14655b7124daa291a9d8d66d)) +* preproccessor ui and expanding preprocessor support ([#6872](https://github.com/windmill-labs/windmill/issues/6872)) ([fc5034e](https://github.com/windmill-labs/windmill/commit/fc5034e94d513bab2feb0ebb2aff822bbb42e4aa)) +* proper error handling in pulled job preprocessor ([#7098](https://github.com/windmill-labs/windmill/issues/7098)) ([84992cd](https://github.com/windmill-labs/windmill/commit/84992cd8ffa94e009aeb6b25db224a3001581e15)) + +## [1.574.1](https://github.com/windmill-labs/windmill/compare/v1.574.0...v1.574.1) (2025-11-08) + + +### Bug Fixes + +* make get_logs work even for partial flow jobs ([d6421c2](https://github.com/windmill-labs/windmill/commit/d6421c2ea79993ef7815c50cf035d3b3425e271e)) + +## [1.574.0](https://github.com/windmill-labs/windmill/compare/v1.573.5...v1.574.0) (2025-11-07) + + +### Features + +* env var in flow ([#6852](https://github.com/windmill-labs/windmill/issues/6852)) ([c59183f](https://github.com/windmill-labs/windmill/commit/c59183f5c39f853d9679c00dd5aa755ef171d735)) + + +### Bug Fixes + +* **cli:** add automatic handler of .node files for codebase bundler ([62ffe9f](https://github.com/windmill-labs/windmill/commit/62ffe9ffce6d070c81794e8483ab83b431aee9ea)) +* teams selector svelte5 ([#7087](https://github.com/windmill-labs/windmill/issues/7087)) ([6045f0c](https://github.com/windmill-labs/windmill/commit/6045f0c40654a88e93be688bcdfab874cfc0b267)) + +## [1.573.5](https://github.com/windmill-labs/windmill/compare/v1.573.4...v1.573.5) (2025-11-07) + + +### Bug Fixes + +* dep jobs improvements ([#7081](https://github.com/windmill-labs/windmill/issues/7081)) ([ad43680](https://github.com/windmill-labs/windmill/commit/ad43680bb979f80db3bdb182ccb5bc61ca385ceb)) +* modal action in App AgGrid ([#7085](https://github.com/windmill-labs/windmill/issues/7085)) ([b07c35f](https://github.com/windmill-labs/windmill/commit/b07c35fe1c9dbc4955ce47766ef92bd5cfdac4c8)) + +## [1.573.4](https://github.com/windmill-labs/windmill/compare/v1.573.3...v1.573.4) (2025-11-07) + + +### Bug Fixes + +* **bun:** support relative imports without .ts but starting with /u/ or /f/ ([9e3ea76](https://github.com/windmill-labs/windmill/commit/9e3ea7657568d6561abcdfd23cc8a210432e4050)) +* create git branch right before creating the workspace fork to catch errors and have a coherent fork point ([#7073](https://github.com/windmill-labs/windmill/issues/7073)) ([192fecc](https://github.com/windmill-labs/windmill/commit/192fecc86f54cf2a69ec7dddc466139b40933ac9)) +* improve priority for child jobs ran from api ([5f7e0b7](https://github.com/windmill-labs/windmill/commit/5f7e0b7244e27533d9a8d1551e8affeb17eb94c0)) +* make it impossible to have non linear script history even in edge condition ([3bda6da](https://github.com/windmill-labs/windmill/commit/3bda6da1683589180cad0780dd87b391a04851f8)) + +## [1.573.3](https://github.com/windmill-labs/windmill/compare/v1.573.2...v1.573.3) (2025-11-06) + + +### Bug Fixes + +* job streaming improvement after compaction ([f16231d](https://github.com/windmill-labs/windmill/commit/f16231d7c9837efa3873034498abc9bfa87575d5)) + +## [1.573.2](https://github.com/windmill-labs/windmill/compare/v1.573.1...v1.573.2) (2025-11-06) + + +### Bug Fixes + +* authentik frontend baseurl field empty when loaded from db ([#7065](https://github.com/windmill-labs/windmill/issues/7065)) ([8cb8650](https://github.com/windmill-labs/windmill/commit/8cb8650460856a1975c6eee429a2771f784a1555)) +* oidc token issue ([75e056b](https://github.com/windmill-labs/windmill/commit/75e056bbce7e00334a36175eed00eb8eba332fbc)) + +## [1.573.1](https://github.com/windmill-labs/windmill/compare/v1.573.0...v1.573.1) (2025-11-05) + + +### Bug Fixes + +* nits hub search and telemetry ([#7063](https://github.com/windmill-labs/windmill/issues/7063)) ([3338a3e](https://github.com/windmill-labs/windmill/commit/3338a3e3c48cbf05c48f0af6754b4c536bcb25f6)) + +## [1.573.0](https://github.com/windmill-labs/windmill/compare/v1.572.2...v1.573.0) (2025-11-05) + + +### Features + +* add groups to user endpoint + set displayname when group created via igroup api ([#7061](https://github.com/windmill-labs/windmill/issues/7061)) ([8667024](https://github.com/windmill-labs/windmill/commit/86670240a773792f4e84cb15af476b9cd2e79c07)) +* hub actions and better search ([#7056](https://github.com/windmill-labs/windmill/issues/7056)) ([1edfdce](https://github.com/windmill-labs/windmill/commit/1edfdce0ddd521819e4cca9dd2e57a20757ad400)) + + +### Bug Fixes + +* incorrect query arg in hub link ([2303f2c](https://github.com/windmill-labs/windmill/commit/2303f2cca0f65bdf69f246cdad2e0fe0bedb3e6e)) +* **lsp:** fix ruff integration ([0271a9f](https://github.com/windmill-labs/windmill/commit/0271a9f51dfdbda950600d41f59ea18c0dafe7a0)) + +## [1.572.2](https://github.com/windmill-labs/windmill/compare/v1.572.1...v1.572.2) (2025-11-04) + + +### Bug Fixes + +* **aiagent:** force structured tool usage for claude ([#7052](https://github.com/windmill-labs/windmill/issues/7052)) ([5ed41c6](https://github.com/windmill-labs/windmill/commit/5ed41c6d132a55713eadae26371a8f8080af1b6d)) +* redirect first time user ([#7057](https://github.com/windmill-labs/windmill/issues/7057)) ([2add278](https://github.com/windmill-labs/windmill/commit/2add2785640c18db65e6642f37e210577f0fb056)) + +## [1.572.1](https://github.com/windmill-labs/windmill/compare/v1.572.0...v1.572.1) (2025-11-04) + + +### Bug Fixes + +* add workspace error handler cache for improved performance ([4a849ca](https://github.com/windmill-labs/windmill/commit/4a849ca9b96e981bbace020fac7ad3123f330ad3)) +* pass whitelist env vars to bun install ([#7047](https://github.com/windmill-labs/windmill/issues/7047)) ([4220582](https://github.com/windmill-labs/windmill/commit/4220582daf23a5367d534c98fb48f0f54ebaab20)) +* preprocessor schema type ([#7049](https://github.com/windmill-labs/windmill/issues/7049)) ([0ae27a3](https://github.com/windmill-labs/windmill/commit/0ae27a3fe88d6aac046137220aab23e8c0d21590)) +* **ruby:** propagate error correctly ([#7046](https://github.com/windmill-labs/windmill/issues/7046)) ([6052714](https://github.com/windmill-labs/windmill/commit/605271483365d9a9f59e7196b7364200ad14d33a)) + +## [1.572.0](https://github.com/windmill-labs/windmill/compare/v1.571.0...v1.572.0) (2025-11-03) + + +### Features + +* **flow:** Add graph diff visualizer ([#6948](https://github.com/windmill-labs/windmill/issues/6948)) ([04d2ef4](https://github.com/windmill-labs/windmill/commit/04d2ef419dfe64c99da55ecead545cb1cc5cf185)) + + +### Bug Fixes + +* **backend:** add 404 error when not found in resource delete endpoints ([#7036](https://github.com/windmill-labs/windmill/issues/7036)) ([aaadf60](https://github.com/windmill-labs/windmill/commit/aaadf60d8f975df66bb9837d8fe21eaeaaf2e70b)) +* consider duckdb as a normal tag ([#7035](https://github.com/windmill-labs/windmill/issues/7035)) ([16317a4](https://github.com/windmill-labs/windmill/commit/16317a471446e444ada4a67d56be36b146c4dfcf)) +* fix rebuild_dependency_map ([#7026](https://github.com/windmill-labs/windmill/issues/7026)) ([f661caf](https://github.com/windmill-labs/windmill/commit/f661caf2b1bb9cb9a2f388684f94cb3055291da5)) +* include missing tags from default/native consts ([#7034](https://github.com/windmill-labs/windmill/issues/7034)) ([c04489c](https://github.com/windmill-labs/windmill/commit/c04489c463f6b277452b7480659fa65cef1ab1ba)) + + +### Performance Improvements + +* parse flow value only if needed ([#7025](https://github.com/windmill-labs/windmill/issues/7025)) ([b5e341f](https://github.com/windmill-labs/windmill/commit/b5e341fde79d6335c0e27d0adbe0febd49a09d36)) + ## [1.571.0](https://github.com/windmill-labs/windmill/compare/v1.570.0...v1.571.0) (2025-11-01) diff --git a/backend/.sqlx/query-0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7.json b/backend/.sqlx/query-0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7.json new file mode 100644 index 0000000000..f91485e855 --- /dev/null +++ b/backend/.sqlx/query-0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args as \"args: sqlx::types::Json>>\" FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "0084c1246d1391d106da2e67a394eafc6695257632406ed9a2111dba1dd106c7" +} diff --git a/backend/.sqlx/query-0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52.json b/backend/.sqlx/query-0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52.json new file mode 100644 index 0000000000..1880104006 --- /dev/null +++ b/backend/.sqlx/query-0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.id\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE r.ping < now() - ($1 || ' seconds')::interval\n AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0186c1058f147e012b8120c342caf8688a6d1643747be3ec4f784c3029a59e52" +} diff --git a/backend/.sqlx/query-0c0f3909b80c35210fc64c685905308621f9135c2c45a2fa0531ea750387da1f.json b/backend/.sqlx/query-0c0f3909b80c35210fc64c685905308621f9135c2c45a2fa0531ea750387da1f.json new file mode 100644 index 0000000000..cec59be959 --- /dev/null +++ b/backend/.sqlx/query-0c0f3909b80c35210fc64c685905308621f9135c2c45a2fa0531ea750387da1f.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n CASE \n WHEN flow_version.id IS NOT NULL THEN\n (flow_version.value -> 'flow_env' -> $3) #> $4\n ELSE\n (root_job.raw_flow -> 'flow_env' -> $3) #> $4\n END AS \"flow_env: sqlx::types::Json>\"\n FROM \n v2_job current_job\n JOIN \n v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id)\n AND root_job.workspace_id = current_job.workspace_id\n LEFT JOIN\n flow_version ON flow_version.id = root_job.runnable_id\n AND flow_version.path = root_job.runnable_path\n AND flow_version.workspace_id = root_job.workspace_id\n WHERE \n current_job.id = $1 AND \n current_job.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flow_env: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0c0f3909b80c35210fc64c685905308621f9135c2c45a2fa0531ea750387da1f" +} diff --git a/backend/.sqlx/query-1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991.json b/backend/.sqlx/query-1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991.json new file mode 100644 index 0000000000..4e4ae9586c --- /dev/null +++ b/backend/.sqlx/query-1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2 RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1d9498226b3d962688558d8ab77f88de5dc8ae5321fa7ff0c7632e628556f991" +} diff --git a/backend/.sqlx/query-1fc43faf3a7ea15be82b1666273836901cc6ba17c7d5e194b50b9204f264de27.json b/backend/.sqlx/query-1fc43faf3a7ea15be82b1666273836901cc6ba17c7d5e194b50b9204f264de27.json new file mode 100644 index 0000000000..d20ff3ba93 --- /dev/null +++ b/backend/.sqlx/query-1fc43faf3a7ea15be82b1666273836901cc6ba17c7d5e194b50b9204f264de27.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1fc43faf3a7ea15be82b1666273836901cc6ba17c7d5e194b50b9204f264de27" +} diff --git a/backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json b/backend/.sqlx/query-24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd.json similarity index 52% rename from backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json rename to backend/.sqlx/query-24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd.json index 58f693df33..7acbf9deec 100644 --- a/backend/.sqlx/query-acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359.json +++ b/backend/.sqlx/query-24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id AS \"id!\" FROM v2_as_queue WHERE id = ANY($1) AND schedule_path IS NULL AND ($2::text[] IS NULL OR tag = ANY($2))", + "query": "SELECT j.id AS \"id!\" FROM v2_job j WHERE j.id = ANY($1) AND j.trigger_kind != 'schedule'::job_trigger_kind AND ($2::text[] IS NULL OR j.tag = ANY($2))", "describe": { "columns": [ { @@ -16,8 +16,8 @@ ] }, "nullable": [ - true + false ] }, - "hash": "acc0b67c8e768b524b5cfb309e4307daeb0c095e07063c57b26ab94211bf6359" + "hash": "24d302b8215d49a289bedd14a5791e9366d1f6d3d2aa485e0f50c6f2d85693dd" } diff --git a/backend/.sqlx/query-255415e4b1e891ce43b92ebde534f116c4ea41ccd1bc994d59c1cd56aae71334.json b/backend/.sqlx/query-255415e4b1e891ce43b92ebde534f116c4ea41ccd1bc994d59c1cd56aae71334.json deleted file mode 100644 index 3daa40769d..0000000000 --- a/backend/.sqlx/query-255415e4b1e891ce43b92ebde534f116c4ea41ccd1bc994d59c1cd56aae71334.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT f.value as \"value: Json\" FROM v2_job j JOIN flow_version f ON f.id = j.runnable_id WHERE j.id = $1 AND j.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "value: Json", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "255415e4b1e891ce43b92ebde534f116c4ea41ccd1bc994d59c1cd56aae71334" -} diff --git a/backend/.sqlx/query-25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364.json b/backend/.sqlx/query-25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364.json new file mode 100644 index 0000000000..7c07d169b8 --- /dev/null +++ b/backend/.sqlx/query-25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT COALESCE(s.flow_status, s.workflow_as_code_status)->'user_states'->$1\n FROM v2_job_queue q LEFT JOIN v2_job_status s USING (id)\n WHERE q.id = $2 AND q.workspace_id = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "25cba74bec5959e6752265cd7b6f84846f74d468f0073f02f81122895e86c364" +} diff --git a/backend/.sqlx/query-2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a.json b/backend/.sqlx/query-2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a.json new file mode 100644 index 0000000000..afa445d679 --- /dev/null +++ b/backend/.sqlx/query-2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now()\n OR (q.suspend_until IS NOT NULL\n AND (q.suspend <= 0 OR q.suspend_until <= now())))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "2729c73b53605908e2fa26b4e4bbbd75d6d5bb77e6a92bf187d978a059d7af4a" +} diff --git a/backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json b/backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json deleted file mode 100644 index e9705c23a6..0000000000 --- a/backend/.sqlx/query-280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT script_path FROM v2_as_completed_job WHERE id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "script_path", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - true - ] - }, - "hash": "280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866" -} diff --git a/backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json b/backend/.sqlx/query-29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99.json similarity index 58% rename from backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json rename to backend/.sqlx/query-29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99.json index 08bb98cc06..11f612ccda 100644 --- a/backend/.sqlx/query-2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9.json +++ b/backend/.sqlx/query-29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\"\n FROM v2_as_completed_job\n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "query": "SELECT j.created_by AS \"created_by!\", j.args as \"args: sqlx::types::Json>\"\n FROM v2_job j\n WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { @@ -22,9 +22,9 @@ ] }, "nullable": [ - true, + false, true ] }, - "hash": "2bfa1ffb3d5869fc3038049ba77890203332e398c865c47aaf019dd5721d59f9" + "hash": "29785f22ee7092e8cebeae3757caf0563e3f98fff1f100616c33e8dc95fbff99" } diff --git a/backend/.sqlx/query-2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666.json b/backend/.sqlx/query-371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b.json similarity index 64% rename from backend/.sqlx/query-2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666.json rename to backend/.sqlx/query-371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b.json index 405902863c..30550d5011 100644 --- a/backend/.sqlx/query-2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666.json +++ b/backend/.sqlx/query-371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT script_path FROM v2_as_queue WHERE id = $1", + "query": "SELECT runnable_path as script_path FROM v2_job WHERE id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ true ] }, - "hash": "2ae44acae7ac80e191b37071baa26e57488489177fe596940d3698d843a93666" + "hash": "371d652fbb1d34d56f645c75d12852ff982821ed7b24574cdd09a06eac0d628b" } diff --git a/backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json b/backend/.sqlx/query-37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950.json similarity index 55% rename from backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json rename to backend/.sqlx/query-37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950.json index ee5e15118d..c26f747c98 100644 --- a/backend/.sqlx/query-2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f.json +++ b/backend/.sqlx/query-37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_as_queue\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id\n WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", + "query": "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_job j\n LEFT JOIN job_logs ON job_logs.job_id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { @@ -32,11 +32,11 @@ ] }, "nullable": [ - true, + false, null, null, true ] }, - "hash": "2ea447f9e644554d415367b91042687ee8690d475b8ed31c48e31180689a278f" + "hash": "37285436c16684449b33810d97d0a2611dab30faff2891fc3a7f00ee8c120950" } diff --git a/backend/.sqlx/query-38baa6cf7d1c2532d38486a01e6d27ac1a58540a51e97dec5613e9bc0791e890.json b/backend/.sqlx/query-38baa6cf7d1c2532d38486a01e6d27ac1a58540a51e97dec5613e9bc0791e890.json new file mode 100644 index 0000000000..eeb0418109 --- /dev/null +++ b/backend/.sqlx/query-38baa6cf7d1c2532d38486a01e6d27ac1a58540a51e97dec5613e9bc0791e890.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_agent_memory (workspace_id, conversation_id, step_id, messages, created_at, updated_at)\n VALUES ($1, $2, $3, $4, NOW(), NOW())\n ON CONFLICT (workspace_id, conversation_id, step_id)\n DO UPDATE SET\n messages = EXCLUDED.messages,\n updated_at = NOW()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid", + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "38baa6cf7d1c2532d38486a01e6d27ac1a58540a51e97dec5613e9bc0791e890" +} diff --git a/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json b/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json deleted file mode 100644 index a97d16b89c..0000000000 --- a/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO app_version\n (app_id, value, created_by, raw_app)\n SELECT app_id, value, created_by, raw_app\n FROM app_version WHERE id = $1\n RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c" -} diff --git a/backend/.sqlx/query-4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73.json b/backend/.sqlx/query-4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73.json new file mode 100644 index 0000000000..c1fd3462f7 --- /dev/null +++ b/backend/.sqlx/query-4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT j.tag as \"tag!\", COUNT(*) as \"count!\"\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE c.started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR j.workspace_id = $2)\n GROUP BY j.tag\n ORDER BY \"count!\" DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "4545e1e0953cde730272353e78044744910a17af23f8201d853f0a4f8404fc73" +} diff --git a/backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json b/backend/.sqlx/query-48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3.json similarity index 73% rename from backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json rename to backend/.sqlx/query-48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3.json index 45a36b4fe3..61c93e6573 100644 --- a/backend/.sqlx/query-7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69.json +++ b/backend/.sqlx/query-48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n script_path, script_hash AS \"script_hash: ScriptHash\",\n job_kind AS \"job_kind!: JobKind\",\n flow_status AS \"flow_status: Json>\",\n raw_flow AS \"raw_flow: Json>\"\n FROM v2_as_completed_job WHERE id = $1 and workspace_id = $2", + "query": "SELECT\n j.runnable_path as script_path, j.runnable_id AS \"script_hash: ScriptHash\",\n j.kind AS \"job_kind!: JobKind\",\n COALESCE(c.flow_status, c.workflow_as_code_status) AS \"flow_status: Json>\",\n j.raw_flow AS \"raw_flow: Json>\"\n FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1 and j.workspace_id = $2", "describe": { "columns": [ { @@ -65,10 +65,10 @@ "nullable": [ true, true, - true, - true, + false, + null, true ] }, - "hash": "7af1cf089022fc1c3597b270b69aa669a153ab0c0bb2807cd4f7fd405afa6f69" + "hash": "48a41e1ad0ad8fc51624af4de34964c72fe4b3cf2497644d514c5c00d6d71bf3" } diff --git a/backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json b/backend/.sqlx/query-49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e.json similarity index 65% rename from backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json rename to backend/.sqlx/query-49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e.json index 07ae8f17c5..d57628b1aa 100644 --- a/backend/.sqlx/query-2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f.json +++ b/backend/.sqlx/query-49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n script_path, args AS \"args: sqlx::types::Json>>\",\n tag AS \"tag!\", priority\n FROM v2_as_completed_job\n WHERE id = $1 and workspace_id = $2", + "query": "SELECT\n j.runnable_path as script_path, j.args AS \"args: sqlx::types::Json>>\",\n j.tag AS \"tag!\", j.priority\n FROM v2_job j\n WHERE j.id = $1 and j.workspace_id = $2", "describe": { "columns": [ { @@ -33,9 +33,9 @@ "nullable": [ true, true, - true, + false, true ] }, - "hash": "2456fc71fc7a0758a4c1fbe77d72fbac2fead0e1bff4e909fd7fb1a41bc35d8f" + "hash": "49f7e7481019d27b44d5c8a167f9d5d3309a31b195b3147fa32f2ca8a6c9c90e" } diff --git a/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json b/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json deleted file mode 100644 index b2c0b4702c..0000000000 --- a/backend/.sqlx/query-519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM v2_as_queue WHERE canceled = false AND (scheduled_for <= now()\n OR (suspend_until IS NOT NULL\n AND ( suspend <= 0\n OR suspend_until <= now())))", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "519f4f76649947f036a2129c11e92ef0ad30e39eec59c27bae8cb0622062c8fb" -} diff --git a/backend/.sqlx/query-528cdbb75f1c5135170a58fce3fda464be138272487639d0ffbbbe6961ec5c37.json b/backend/.sqlx/query-528cdbb75f1c5135170a58fce3fda464be138272487639d0ffbbbe6961ec5c37.json deleted file mode 100644 index e998bea512..0000000000 --- a/backend/.sqlx/query-528cdbb75f1c5135170a58fce3fda464be138272487639d0ffbbbe6961ec5c37.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE job_logs SET logs = $1, log_offset = $2, \n log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3) \n WHERE workspace_id = $4 AND job_id = $5", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int4", - "Text", - "Text", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "528cdbb75f1c5135170a58fce3fda464be138272487639d0ffbbbe6961ec5c37" -} diff --git a/backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json b/backend/.sqlx/query-57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825.json similarity index 50% rename from backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json rename to backend/.sqlx/query-57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825.json index 0a9e91b206..1bb262acbf 100644 --- a/backend/.sqlx/query-0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd.json +++ b/backend/.sqlx/query-57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now() AND ($3::text[] IS NULL OR tag = ANY($3))", + "query": "SELECT coalesce(COUNT(*) FILTER(WHERE q.suspend = 0 AND q.running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE q.suspend > 0), 0) as \"suspended!\" FROM v2_job_queue q JOIN v2_job j USING (id) WHERE (j.workspace_id = $1 OR $2) AND q.scheduled_for <= now() AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { @@ -26,5 +26,5 @@ null ] }, - "hash": "0cb0e912bc942af2b1ef784455f3f073a79e300f3dd48f14122d1782eee663cd" + "hash": "57bd445bfdc667089f847acb9dc484028ce8a077308979a5ba8ef252e19aa825" } diff --git a/backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json b/backend/.sqlx/query-5e7cadffbee74b11e224b60322b102b9899b4a97b8e557692c0085a9b472b8a7.json similarity index 52% rename from backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json rename to backend/.sqlx/query-5e7cadffbee74b11e224b60322b102b9899b4a97b8e557692c0085a9b472b8a7.json index a76515fd7e..414ba5924d 100644 --- a/backend/.sqlx/query-ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9.json +++ b/backend/.sqlx/query-5e7cadffbee74b11e224b60322b102b9899b4a97b8e557692c0085a9b472b8a7.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", + "query": "SELECT j.created_by AS \"created_by\", coalesce(job_logs.logs, '') as logs, COALESCE(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM v2_job j\n LEFT JOIN job_logs ON job_logs.job_id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", "describe": { "columns": [ { "ordinal": 0, - "name": "created_by!", + "name": "created_by", "type_info": "Varchar" }, { @@ -32,11 +32,11 @@ ] }, "nullable": [ - true, - null, false, + null, + null, true ] }, - "hash": "ca5d9a9d8d18da970c7fd6eab41ecbb3a5c7c29803e4c38b8a0b2ca3790e52f9" + "hash": "5e7cadffbee74b11e224b60322b102b9899b4a97b8e557692c0085a9b472b8a7" } diff --git a/backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json b/backend/.sqlx/query-6a1005b8ae5326c5d5955534e9228b6bd524253971ae09cbe100c0d6d42e6cdc.json similarity index 50% rename from backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json rename to backend/.sqlx/query-6a1005b8ae5326c5d5955534e9228b6bd524253971ae09cbe100c0d6d42e6cdc.json index 804f35777e..647e56619c 100644 --- a/backend/.sqlx/query-f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0.json +++ b/backend/.sqlx/query-6a1005b8ae5326c5d5955534e9228b6bd524253971ae09cbe100c0d6d42e6cdc.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT flow_status->'user_states'->$1\n FROM v2_as_queue\n WHERE id = $2 AND workspace_id = $3\n ", + "query": "SELECT messages FROM ai_agent_memory\n WHERE workspace_id = $1 AND conversation_id = $2 AND step_id = $3", "describe": { "columns": [ { "ordinal": 0, - "name": "?column?", + "name": "messages", "type_info": "Jsonb" } ], @@ -17,8 +17,8 @@ ] }, "nullable": [ - null + false ] }, - "hash": "f3571e1d2b57011e5f6a38725eb42d909456d28a98563923cca43e760862e5e0" + "hash": "6a1005b8ae5326c5d5955534e9228b6bd524253971ae09cbe100c0d6d42e6cdc" } diff --git a/backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json b/backend/.sqlx/query-6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76.json similarity index 57% rename from backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json rename to backend/.sqlx/query-6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76.json index 3422605ef1..b2e0c68ba5 100644 --- a/backend/.sqlx/query-d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45.json +++ b/backend/.sqlx/query-6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id AS \"id!\", flow_status, suspend AS \"suspend!\", script_path\n FROM v2_as_queue\n WHERE id = $1\n ", + "query": "\n SELECT j.id AS \"id!\", COALESCE(s.flow_status, s.workflow_as_code_status) as flow_status, q.suspend AS \"suspend!\", j.runnable_path as script_path\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE j.id = $1\n ", "describe": { "columns": [ { @@ -30,11 +30,11 @@ ] }, "nullable": [ - true, - true, - true, + false, + null, + false, true ] }, - "hash": "d4d83d8177144c91aa489b5a42a45c83f8b069a52f681f14afb4931ac77baf45" + "hash": "6b9ea0059ad1037a77f67b70d5de13776870612e8faa8d8fa577f8e7b7309f76" } diff --git a/backend/.sqlx/query-6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750.json b/backend/.sqlx/query-6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750.json new file mode 100644 index 0000000000..4b9bfe326b --- /dev/null +++ b/backend/.sqlx/query-6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750.json @@ -0,0 +1,216 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as, \n j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: JobTriggerKind\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl\n FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id\n WHERE j.id = $1 AND j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "runnable_id: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 6, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 7, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent" + ] + } + } + } + }, + { + "ordinal": 9, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 12, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "trigger_kind: JobTriggerKind", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 17, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 18, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 19, + "name": "cache_ttl", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + true, + false, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + true + ] + }, + "hash": "6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750" +} diff --git a/backend/.sqlx/query-6ec1e81e1698c754ca8a660cf6a3ae66e75b487901befd647839dd2bc2233bf4.json b/backend/.sqlx/query-6ec1e81e1698c754ca8a660cf6a3ae66e75b487901befd647839dd2bc2233bf4.json new file mode 100644 index 0000000000..6321ca9156 --- /dev/null +++ b/backend/.sqlx/query-6ec1e81e1698c754ca8a660cf6a3ae66e75b487901befd647839dd2bc2233bf4.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_logs SET logs = $1 WHERE workspace_id = $2 AND job_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "6ec1e81e1698c754ca8a660cf6a3ae66e75b487901befd647839dd2bc2233bf4" +} diff --git a/backend/.sqlx/query-e8e33f599eae064011232f9f715e676d0c8ae31982865cb2c8103ed735c42c69.json b/backend/.sqlx/query-6f49a47b0b77720b0e8a2ddbd46e45e414dd646c912387ee63609d9f5611690f.json similarity index 54% rename from backend/.sqlx/query-e8e33f599eae064011232f9f715e676d0c8ae31982865cb2c8103ed735c42c69.json rename to backend/.sqlx/query-6f49a47b0b77720b0e8a2ddbd46e45e414dd646c912387ee63609d9f5611690f.json index 84645b71b0..f7cad58326 100644 --- a/backend/.sqlx/query-e8e33f599eae064011232f9f715e676d0c8ae31982865cb2c8103ed735c42c69.json +++ b/backend/.sqlx/query-6f49a47b0b77720b0e8a2ddbd46e45e414dd646c912387ee63609d9f5611690f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL DO UPDATE SET callback_job_ids = EXCLUDED.callback_job_ids, deployment_msg = EXCLUDED.deployment_msg", + "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL DO UPDATE SET callback_job_ids = EXCLUDED.callback_job_ids, deployment_msg = EXCLUDED.deployment_msg", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "e8e33f599eae064011232f9f715e676d0c8ae31982865cb2c8103ed735c42c69" + "hash": "6f49a47b0b77720b0e8a2ddbd46e45e414dd646c912387ee63609d9f5611690f" } diff --git a/backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json b/backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json new file mode 100644 index 0000000000..7cfd1e5b64 --- /dev/null +++ b/backend/.sqlx/query-70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n j.id AS \"id!\", j.workspace_id AS \"workspace_id!\", j.parent_job, j.flow_step_id IS NOT NULL AS \"is_flow_step?\",\n COALESCE(s.flow_status, s.workflow_as_code_status) AS \"flow_status: Box\", r.ping AS last_ping, j.same_worker AS \"same_worker?\"\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now()\n AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode')\n AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval\n AND q.canceled_by IS NULL\n \n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "is_flow_step?", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "flow_status: Box", + "type_info": "Jsonb" + }, + { + "ordinal": 5, + "name": "last_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "same_worker?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + true, + null, + null, + true, + false + ] + }, + "hash": "70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b" +} diff --git a/backend/.sqlx/query-72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43.json b/backend/.sqlx/query-72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43.json deleted file mode 100644 index 2ffdf141b1..0000000000 --- a/backend/.sqlx/query-72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n id AS \"id!\", workspace_id AS \"workspace_id!\", parent_job, is_flow_step,\n flow_status AS \"flow_status: Box\", last_ping, same_worker\n FROM v2_as_queue\n WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now()\n AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode')\n AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval\n AND canceled = false\n \n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "workspace_id!", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "parent_job", - "type_info": "Uuid" - }, - { - "ordinal": 3, - "name": "is_flow_step", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "flow_status: Box", - "type_info": "Jsonb" - }, - { - "ordinal": 5, - "name": "last_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "same_worker", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "72956f508f66312807738399b57aa01a048fa4f9281327cf5b78111178424b43" -} diff --git a/backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json b/backend/.sqlx/query-7a4e79bcae7c7d69e4bb71c8540b2adb9ca8bd15098038e3ccf8b16164944951.json similarity index 58% rename from backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json rename to backend/.sqlx/query-7a4e79bcae7c7d69e4bb71c8540b2adb9ca8bd15098038e3ccf8b16164944951.json index ef26a8453e..1eb38e8623 100644 --- a/backend/.sqlx/query-6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c.json +++ b/backend/.sqlx/query-7a4e79bcae7c7d69e4bb71c8540b2adb9ca8bd15098038e3ccf8b16164944951.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) FROM v2_as_queue WHERE email = $1", + "query": "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE j.permissioned_as_email = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "6ab112fa42a9ae332bfa30427b70fa742351c5c180ac3de106df54f7badb494c" + "hash": "7a4e79bcae7c7d69e4bb71c8540b2adb9ca8bd15098038e3ccf8b16164944951" } diff --git a/backend/.sqlx/query-83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604.json b/backend/.sqlx/query-83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604.json new file mode 100644 index 0000000000..27a5df6de9 --- /dev/null +++ b/backend/.sqlx/query-83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app_version\n (app_id, value, created_by, raw_app)\n SELECT app_id, value, created_by, raw_app\n FROM app_version WHERE id = $1\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604" +} diff --git a/backend/.sqlx/query-85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c.json b/backend/.sqlx/query-85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c.json new file mode 100644 index 0000000000..5573e5208b --- /dev/null +++ b/backend/.sqlx/query-85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "85ec6c6384ab77df102d05caf82079e65a5300a52a292b9143b755087b9f9c4c" +} diff --git a/backend/.sqlx/query-89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17.json b/backend/.sqlx/query-89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17.json deleted file mode 100644 index 25dd18003c..0000000000 --- a/backend/.sqlx/query-89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT usr.email, usage.executions\n FROM usr\n , LATERAL (\n SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_as_completed_job\n WHERE workspace_id = $1\n AND job_kind NOT IN ('flow', 'flowpreview', 'flownode')\n AND email = usr.email\n AND now() - '1 week'::interval < created_at\n ) usage\n WHERE workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "executions", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "89940a53f29b173b6a8717f057a81a4f2edad152581950fdd80d758a0d242c17" -} diff --git a/backend/.sqlx/query-2e131a019051bdac7c9c65f7c504cfba31cfdd64a0f68001e123694ed5cde5ed.json b/backend/.sqlx/query-8e750d4b3af9b5844c11b1b92741f2ee9d2d3412d4a2c96c6ccc87ec1c382384.json similarity index 55% rename from backend/.sqlx/query-2e131a019051bdac7c9c65f7c504cfba31cfdd64a0f68001e123694ed5cde5ed.json rename to backend/.sqlx/query-8e750d4b3af9b5844c11b1b92741f2ee9d2d3412d4a2c96c6ccc87ec1c382384.json index 347ad02c31..e355026cf4 100644 --- a/backend/.sqlx/query-2e131a019051bdac7c9c65f7c504cfba31cfdd64a0f68001e123694ed5cde5ed.json +++ b/backend/.sqlx/query-8e750d4b3af9b5844c11b1b92741f2ee9d2d3412d4a2c96c6ccc87ec1c382384.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5) \n ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET callback_job_ids = EXCLUDED.callback_job_ids, deployment_msg = EXCLUDED.deployment_msg", + "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5) \n ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET callback_job_ids = EXCLUDED.callback_job_ids, deployment_msg = EXCLUDED.deployment_msg", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "2e131a019051bdac7c9c65f7c504cfba31cfdd64a0f68001e123694ed5cde5ed" + "hash": "8e750d4b3af9b5844c11b1b92741f2ee9d2d3412d4a2c96c6ccc87ec1c382384" } diff --git a/backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json b/backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json deleted file mode 100644 index 61598c0b2d..0000000000 --- a/backend/.sqlx/query-90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT tag as \"tag!\", COUNT(*) as \"count!\"\n FROM v2_as_completed_job\n WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)\n GROUP BY tag\n ORDER BY \"count!\" DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "count!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Float8", - "Text" - ] - }, - "nullable": [ - true, - null - ] - }, - "hash": "90fbb9430ab03ce3aadd95cc263e5a3d1a91ea02de7608676575e1c03023ed71" -} diff --git a/backend/.sqlx/query-99c289a8bcf87588ecb89575f66e1fbfce74dd6b69a8a039714a02ce7558a1b3.json b/backend/.sqlx/query-99c289a8bcf87588ecb89575f66e1fbfce74dd6b69a8a039714a02ce7558a1b3.json new file mode 100644 index 0000000000..6e9dd142f6 --- /dev/null +++ b/backend/.sqlx/query-99c289a8bcf87588ecb89575f66e1fbfce74dd6b69a8a039714a02ce7558a1b3.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT instance_group.id, COALESCE(instance_group.scim_display_name, instance_group.name) as display_name\n FROM email_to_igroup\n JOIN instance_group ON instance_group.name = email_to_igroup.igroup\n WHERE email_to_igroup.email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "display_name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + null + ] + }, + "hash": "99c289a8bcf87588ecb89575f66e1fbfce74dd6b69a8a039714a02ce7558a1b3" +} diff --git a/backend/.sqlx/query-a0af7dc507778ab23d5b1615e2a03278de05f4cd90fb1c31d7281f8681c7f85d.json b/backend/.sqlx/query-a0af7dc507778ab23d5b1615e2a03278de05f4cd90fb1c31d7281f8681c7f85d.json new file mode 100644 index 0000000000..2741840d4b --- /dev/null +++ b/backend/.sqlx/query-a0af7dc507778ab23d5b1615e2a03278de05f4cd90fb1c31d7281f8681c7f85d.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_logs SET logs = $1, log_offset = $2,\n log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3)\n WHERE workspace_id = $4 AND job_id = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int4", + "Text", + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a0af7dc507778ab23d5b1615e2a03278de05f4cd90fb1c31d7281f8681c7f85d" +} diff --git a/backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json b/backend/.sqlx/query-a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe.json similarity index 55% rename from backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json rename to backend/.sqlx/query-a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe.json index 1afb036030..d66a3aa9f0 100644 --- a/backend/.sqlx/query-11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a.json +++ b/backend/.sqlx/query-a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) FROM v2_as_queue WHERE running = true AND email = $1", + "query": "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE q.running = true AND j.permissioned_as_email = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "11db65c493990f6935103033b2fbb0c08ae6d91b05b2f3f7c89a990d1d5a5f8a" + "hash": "a3d18ae5e5125940ae0d6af315e2dbb17c739415850e7c67f8575fb983c295fe" } diff --git a/backend/.sqlx/query-a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0.json b/backend/.sqlx/query-a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0.json deleted file mode 100644 index 1c396d38fd..0000000000 --- a/backend/.sqlx/query-a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO flow_version\n (workspace_id, path, value, schema, created_by)\n\n SELECT workspace_id, path, value, schema, created_by\n FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3\n\n RETURNING id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0" -} diff --git a/backend/.sqlx/query-a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d.json b/backend/.sqlx/query-a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d.json new file mode 100644 index 0000000000..af4ada67f2 --- /dev/null +++ b/backend/.sqlx/query-a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $2\n AND (j.kind = 'appscript' OR j.kind = 'preview')\n AND j.created_by = 'anonymous'\n AND c.started_at > now() - interval '3 hours'\n AND j.runnable_path LIKE $3 || '/%'\n AND c.result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a72e7fb55d7268fe1ea40015a4a84b12dd961ba732772d8f6c59d18fe05f285d" +} diff --git a/backend/.sqlx/query-be00ac55e8668a0ed3befda7d8595c7cda0cba0b119d4fdb8a0dea1b28a1d560.json b/backend/.sqlx/query-be00ac55e8668a0ed3befda7d8595c7cda0cba0b119d4fdb8a0dea1b28a1d560.json new file mode 100644 index 0000000000..182fa4e788 --- /dev/null +++ b/backend/.sqlx/query-be00ac55e8668a0ed3befda7d8595c7cda0cba0b119d4fdb8a0dea1b28a1d560.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n (elem->>'installation_id')::bigint as installation_id\n FROM workspace_settings,\n LATERAL jsonb_array_elements(git_app_installations) AS elem\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "installation_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "be00ac55e8668a0ed3befda7d8595c7cda0cba0b119d4fdb8a0dea1b28a1d560" +} diff --git a/backend/.sqlx/query-d1d9cb0a7043760564171f481dd73f04eaa2c7d5c1ddc0501bc70c3fb34a07be.json b/backend/.sqlx/query-d1d9cb0a7043760564171f481dd73f04eaa2c7d5c1ddc0501bc70c3fb34a07be.json new file mode 100644 index 0000000000..2243c6f719 --- /dev/null +++ b/backend/.sqlx/query-d1d9cb0a7043760564171f481dd73f04eaa2c7d5c1ddc0501bc70c3fb34a07be.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_agent_memory\n WHERE workspace_id = $1 AND conversation_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "d1d9cb0a7043760564171f481dd73f04eaa2c7d5c1ddc0501bc70c3fb34a07be" +} diff --git a/backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json b/backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json new file mode 100644 index 0000000000..73fb2e98d8 --- /dev/null +++ b/backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json @@ -0,0 +1,239 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, q.workspace_id, j.runnable_id as \"runnable_id: ScriptHash\", scheduled_for, parent_job, flow_innermost_root_job, runnable_path, kind as \"kind: JobKind\", started_at, permissioned_as, created_by, script_lang as \"script_lang: ScriptLang\", \n permissioned_as_email, flow_step_id, trigger_kind as \"trigger_kind: JobTriggerKind\", trigger, q.priority, concurrent_limit, q.tag, cache_ttl, r.ping as last_ping, worker, memory_peak, running\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE j.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "runnable_id: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 5, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 6, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "kind: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent" + ] + } + } + } + }, + { + "ordinal": 8, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 12, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "trigger_kind: JobTriggerKind", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 17, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 18, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 19, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 20, + "name": "last_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 21, + "name": "worker", + "type_info": "Varchar" + }, + { + "ordinal": 22, + "name": "memory_peak", + "type_info": "Int4" + }, + { + "ordinal": 23, + "name": "running", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + false, + true, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd" +} diff --git a/backend/.sqlx/query-d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856.json b/backend/.sqlx/query-d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856.json deleted file mode 100644 index 87361665b5..0000000000 --- a/backend/.sqlx/query-d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "d5d97005eebcb2760216dbd10872acdb42868fd5f7a27f71836e7a6ff1c69856" -} diff --git a/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json b/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json deleted file mode 100644 index d87e680abe..0000000000 --- a/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS (\n SELECT 1 FROM v2_as_completed_job\n WHERE workspace_id = $2\n AND (job_kind = 'appscript' OR job_kind = 'preview')\n AND created_by = 'anonymous'\n AND started_at > now() - interval '3 hours'\n AND script_path LIKE $3 || '/%'\n AND result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb\n )", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b" -} diff --git a/backend/.sqlx/query-edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd.json b/backend/.sqlx/query-edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd.json new file mode 100644 index 0000000000..f67e52ab6b --- /dev/null +++ b/backend/.sqlx/query-edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "executions", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "edd57b3d59ddc21b99212b5fabd4a8b793c4ae618a04220b3609c8c3c168f8fd" +} diff --git a/backend/.sqlx/query-f0efa383f2025158de160577ad839ae72faf0c8fe097e6ad6d309aee9a8aede2.json b/backend/.sqlx/query-f0efa383f2025158de160577ad839ae72faf0c8fe097e6ad6d309aee9a8aede2.json new file mode 100644 index 0000000000..32616f298f --- /dev/null +++ b/backend/.sqlx/query-f0efa383f2025158de160577ad839ae72faf0c8fe097e6ad6d309aee9a8aede2.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow_version\n (workspace_id, path, value, schema, created_by)\n\n SELECT workspace_id, path, value, schema, created_by\n FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3\n\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f0efa383f2025158de160577ad839ae72faf0c8fe097e6ad6d309aee9a8aede2" +} diff --git a/backend/.sqlx/query-e572fa64eec9188368d7c271ac7ecd6b45dc161423abab22cddb2b13f6fb9833.json b/backend/.sqlx/query-f844026597d2e1c59a80dc8e1ddd069c0ad82b27b41171825928a938cfa8316f.json similarity index 54% rename from backend/.sqlx/query-e572fa64eec9188368d7c271ac7ecd6b45dc161423abab22cddb2b13f6fb9833.json rename to backend/.sqlx/query-f844026597d2e1c59a80dc8e1ddd069c0ad82b27b41171825928a938cfa8316f.json index 2ee8c422fd..2aab93c8bb 100644 --- a/backend/.sqlx/query-e572fa64eec9188368d7c271ac7ecd6b45dc161423abab22cddb2b13f6fb9833.json +++ b/backend/.sqlx/query-f844026597d2e1c59a80dc8e1ddd069c0ad82b27b41171825928a938cfa8316f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET callback_job_ids = EXCLUDED.callback_job_ids, deployment_msg = EXCLUDED.deployment_msg", + "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET callback_job_ids = EXCLUDED.callback_job_ids, deployment_msg = EXCLUDED.deployment_msg", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "e572fa64eec9188368d7c271ac7ecd6b45dc161423abab22cddb2b13f6fb9833" + "hash": "f844026597d2e1c59a80dc8e1ddd069c0ad82b27b41171825928a938cfa8316f" } diff --git a/backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json b/backend/.sqlx/query-fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e.json similarity index 52% rename from backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json rename to backend/.sqlx/query-fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e.json index 9cda5cc93e..29cedfbc1c 100644 --- a/backend/.sqlx/query-3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e.json +++ b/backend/.sqlx/query-fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH job_info AS (\n -- Query for Teams (running jobs)\n SELECT\n parent.job_kind AS \"job_kind!: JobKind\",\n parent.script_hash AS \"script_hash: ScriptHash\",\n parent.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n child.parent_job AS \"parent_job: Uuid\",\n parent.created_at AS \"created_at!: chrono::NaiveDateTime\",\n parent.created_by AS \"created_by!\",\n parent.script_path,\n parent.args AS \"args: sqlx::types::Json>\"\n FROM v2_as_queue child\n JOIN v2_as_queue parent ON parent.id = child.parent_job\n WHERE child.id = $1 AND child.workspace_id = $2\n UNION ALL\n -- Query for Slack (completed jobs)\n SELECT\n v2_as_queue.job_kind AS \"job_kind!: JobKind\",\n v2_as_queue.script_hash AS \"script_hash: ScriptHash\",\n v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n v2_as_completed_job.parent_job AS \"parent_job: Uuid\",\n v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\",\n v2_as_completed_job.created_by AS \"created_by!\",\n v2_as_queue.script_path,\n v2_as_queue.args AS \"args: sqlx::types::Json>\"\n FROM v2_as_queue\n JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2\n )\n SELECT * FROM job_info LIMIT 1", + "query": "WITH job_info AS (\n -- Query for Teams (running jobs)\n SELECT\n parent_j.kind AS \"job_kind!: JobKind\",\n parent_j.runnable_id AS \"script_hash: ScriptHash\",\n parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n child_j.parent_job AS \"parent_job: Uuid\",\n parent_j.created_at AS \"created_at!: chrono::NaiveDateTime\",\n parent_j.created_by AS \"created_by!\",\n parent_j.runnable_path as script_path,\n parent_j.args AS \"args: sqlx::types::Json>\"\n FROM v2_job_queue child_q JOIN v2_job child_j USING (id)\n JOIN v2_job parent_j ON parent_j.id = child_j.parent_job\n WHERE child_j.id = $1 AND child_j.workspace_id = $2\n UNION ALL\n -- Query for Slack (completed jobs)\n SELECT\n parent_j.kind AS \"job_kind!: JobKind\",\n parent_j.runnable_id AS \"script_hash: ScriptHash\",\n parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\",\n completed_j.parent_job AS \"parent_job: Uuid\",\n completed_j.created_at AS \"created_at!: chrono::NaiveDateTime\",\n completed_j.created_by AS \"created_by!\",\n parent_j.runnable_path as script_path,\n parent_j.args AS \"args: sqlx::types::Json>\"\n FROM v2_job_completed completed_c JOIN v2_job completed_j USING (id)\n JOIN v2_job parent_j ON parent_j.id = completed_j.parent_job\n WHERE completed_j.id = $1 AND completed_j.workspace_id = $2\n )\n SELECT * FROM job_info LIMIT 1", "describe": { "columns": [ { @@ -88,5 +88,5 @@ null ] }, - "hash": "3f08ffbb5c71b873a9e164ecb0b10fffb37599f2a703885ee723cb9290fed13e" + "hash": "fbd38c9c4f4ecba2d3e3d79433327fd4280466eec443aff4f1e8c1ab810cca7e" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 6a8ec39269..d224f52b22 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -556,7 +556,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -670,7 +670,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -692,7 +692,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -703,7 +703,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -788,9 +788,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.8" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf26925f4a5b59eb76722b63c2892b1d70d06fa053c72e4a100ec308c1d47bc" +checksum = "86590e57ea40121d47d3f2e131bfd873dea15d78dc2f4604f4734537ad9e56c4" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -1026,7 +1026,7 @@ dependencies = [ "http 1.3.1", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-rustls 0.24.2", "hyper-rustls 0.27.7", "hyper-util", @@ -1182,7 +1182,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "itoa", "matchit", @@ -1233,7 +1233,7 @@ checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -1373,7 +1373,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.108", + "syn 2.0.110", "which 4.4.2", ] @@ -1394,7 +1394,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -1414,7 +1414,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -1569,7 +1569,7 @@ dependencies = [ "hex", "http 1.3.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -1621,7 +1621,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -1644,7 +1644,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -1654,7 +1654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17d4f95e880cfd28c4ca5a006cf7f6af52b4bcb7b5866f573b2faa126fb7affb" dependencies = [ "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -1788,7 +1788,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -1931,7 +1931,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b4a6cae9efc04cc6cbb8faf338d2c497c165c83e74509cf4dbedea948bbf6e5" dependencies = [ "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -1945,9 +1945,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.44" +version = "1.2.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3" +checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" dependencies = [ "find-msvc-tools", "jobserver", @@ -2083,7 +2083,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -2138,7 +2138,7 @@ dependencies = [ "nom 7.1.3", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -2457,9 +2457,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "rand_core 0.6.4", @@ -2520,7 +2520,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -2613,7 +2613,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -2627,7 +2627,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -2660,7 +2660,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -2671,7 +2671,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -3163,7 +3163,7 @@ checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" dependencies = [ "datafusion-expr", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -3601,7 +3601,7 @@ checksum = "babccedee31ce7e57c3e6dff2cb3ab8d68c49d0df8222fe0d11d628e65192790" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -3625,7 +3625,7 @@ dependencies = [ "hickory-resolver", "http 1.3.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-rustls 0.27.7", "hyper-util", "ipnet", @@ -3713,7 +3713,7 @@ dependencies = [ "http 1.3.1", "httparse", "hyper 0.14.32", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "itertools 0.10.5", "memmem", @@ -3906,7 +3906,7 @@ dependencies = [ "hkdf", "http 1.3.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "idna", "indexmap 2.11.1", @@ -3992,7 +3992,7 @@ dependencies = [ "stringcase", "strum 0.25.0", "strum_macros 0.25.3", - "syn 2.0.108", + "syn 2.0.110", "thiserror 2.0.17", ] @@ -4177,7 +4177,7 @@ dependencies = [ "http 1.3.1", "http-body-util", "hyper 0.14.32", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "libc", "log", @@ -4231,7 +4231,7 @@ dependencies = [ "deno_error", "deno_tls", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-rustls 0.27.7", "hyper-util", "log", @@ -4249,9 +4249,9 @@ dependencies = [ [[package]] name = "deno_terminal" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f71c27009e0141dedd315f1dfa3ebb0a6ca4acce7c080fac576ea415a465f6" +checksum = "f3ba8041ae7319b3ca6a64c399df4112badcbbe0868b4517637647614bede4be" dependencies = [ "once_cell", "termcolor", @@ -4361,7 +4361,7 @@ dependencies = [ "h2 0.4.12", "http 1.3.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "once_cell", "rustls-tokio-stream", @@ -4494,7 +4494,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -4515,7 +4515,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -4559,7 +4559,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -4579,7 +4579,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -4705,7 +4705,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -4740,7 +4740,7 @@ checksum = "788160fb30de9cdd857af31c6a2675904b16ece8fc2737b2c7127ba368c9d0f4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -4950,7 +4950,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -5016,7 +5016,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -5036,7 +5036,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -5056,7 +5056,7 @@ checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -5144,6 +5144,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "eventsource-stream" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab" +dependencies = [ + "futures-core", + "nom 7.1.3", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -5214,7 +5225,7 @@ dependencies = [ "base64 0.21.7", "bytes", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "pin-project", "rand 0.8.5", @@ -5373,7 +5384,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -5411,7 +5422,7 @@ checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" dependencies = [ "proc-macro2", "swc_macros_common", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -5546,7 +5557,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -5843,9 +5854,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -5931,7 +5942,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -6263,7 +6274,7 @@ checksum = "149e3ea90eb5a26ad354cfe3cb7f7401b9329032d0235f2687d03a35f30e5d4c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -6593,9 +6604,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +checksum = "1744436df46f0bde35af3eda22aeaba453aada65d8f1c171cd8a5f59030bd69f" dependencies = [ "atomic-waker", "bytes", @@ -6624,7 +6635,7 @@ dependencies = [ "futures-util", "headers", "http 1.3.1", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-rustls 0.27.7", "hyper-util", "pin-project-lite", @@ -6641,7 +6652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "pin-project-lite", "tokio", @@ -6672,7 +6683,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.3.1", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "log", "rustls 0.23.29", @@ -6681,7 +6692,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tower-service", - "webpki-roots 1.0.3", + "webpki-roots 1.0.4", ] [[package]] @@ -6690,7 +6701,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "pin-project-lite", "tokio", @@ -6718,7 +6729,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "native-tls", "tokio", @@ -6739,7 +6750,7 @@ dependencies = [ "futures-util", "http 1.3.1", "http-body 1.0.1", - "hyper 1.7.0", + "hyper 1.8.0", "ipnet", "libc", "percent-encoding", @@ -6760,7 +6771,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "pin-project-lite", "tokio", @@ -7054,9 +7065,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344" dependencies = [ "bitflags 2.9.4", "cfg-if", @@ -7092,9 +7103,9 @@ dependencies = [ [[package]] name = "iri-string" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" dependencies = [ "memchr", "serde", @@ -7109,7 +7120,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -7408,7 +7419,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-http-proxy", "hyper-rustls 0.27.7", "hyper-timeout", @@ -7460,7 +7471,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -7516,7 +7527,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -7713,9 +7724,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.22" +version = "1.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" dependencies = [ "cc", "libc", @@ -8123,7 +8134,7 @@ checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -8242,7 +8253,7 @@ checksum = "c402a4092d5e204f32c9e155431046831fa712637043c58cb73bc6bc6c9663b5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -8287,7 +8298,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", "termcolor", "thiserror 2.0.17", ] @@ -8390,7 +8401,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -8596,7 +8607,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -8753,11 +8764,10 @@ dependencies = [ [[package]] name = "num-bigint-dig" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" dependencies = [ - "byteorder", "lazy_static", "libm", "num-integer", @@ -8865,7 +8875,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -8936,7 +8946,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.7.0", + "hyper 1.8.0", "itertools 0.14.0", "md-5 0.10.6", "parking_lot 0.12.5", @@ -9072,9 +9082,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.74" +version = "0.10.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24ad14dd45412269e1a30f52ad8f0664f0f4f4a89ee8fe28c3b3527021ebb654" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ "bitflags 2.9.4", "cfg-if", @@ -9093,7 +9103,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -9113,9 +9123,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.110" +version = "0.9.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a9f0075ba3c21b09f8e8b2026584b1d18d49388648f2fbbf3c97ea8deced8e2" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" dependencies = [ "cc", "libc", @@ -9557,7 +9567,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -9638,7 +9648,7 @@ dependencies = [ "phf_shared 0.11.3", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -9688,7 +9698,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -9900,7 +9910,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -9964,7 +9974,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -9975,7 +9985,7 @@ checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" dependencies = [ "proc-macro-rules-macros", "proc-macro2", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -9987,7 +9997,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -10098,7 +10108,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.108", + "syn 2.0.110", "tempfile", ] @@ -10112,7 +10122,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -10297,9 +10307,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.41" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] @@ -10522,7 +10532,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -10591,7 +10601,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -10702,7 +10712,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-rustls 0.27.7", "hyper-tls 0.6.0", "hyper-util", @@ -10733,7 +10743,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.3", + "webpki-roots 1.0.4", ] [[package]] @@ -10762,7 +10772,7 @@ dependencies = [ "futures", "getrandom 0.2.16", "http 1.3.1", - "hyper 1.7.0", + "hyper 1.8.0", "parking_lot 0.11.2", "reqwest 0.12.24", "reqwest-middleware", @@ -10867,9 +10877,9 @@ dependencies = [ [[package]] name = "rmcp" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fdad1258f7259fdc0f2dfc266939c82c3b5d1fd72bcde274d600cdc27e60243" +checksum = "e5947688160b56fb6c827e3c20a72c90392a1d7e9dec74749197aa1780ac42ca" dependencies = [ "base64 0.22.1", "bytes", @@ -10883,7 +10893,7 @@ dependencies = [ "rand 0.9.0", "reqwest 0.12.24", "rmcp-macros", - "schemars 1.0.4", + "schemars 1.1.0", "serde", "serde_json", "sse-stream", @@ -10898,15 +10908,15 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede0589a208cc7ce81d1be68aa7e74b917fcd03c81528408bab0457e187dcd9b" +checksum = "01263441d3f8635c628e33856c468b96ebbce1af2d3699ea712ca71432d4ee7a" dependencies = [ "darling 0.21.3", "proc-macro2", "quote", "serde_json", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -10996,7 +11006,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.108", + "syn 2.0.110", "walkdir", ] @@ -11463,14 +11473,14 @@ dependencies = [ [[package]] name = "schemars" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" +checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" dependencies = [ "chrono", "dyn-clone", "ref-cast", - "schemars_derive 1.0.4", + "schemars_derive 1.1.0", "serde", "serde_json", ] @@ -11484,19 +11494,19 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] name = "schemars_derive" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d020396d1d138dc19f1165df7545479dcd58d93810dc5d646a16e55abefa80" +checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -11685,7 +11695,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -11696,7 +11706,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -11748,7 +11758,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -11798,7 +11808,7 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.11.1", "schemars 0.9.0", - "schemars 1.0.4", + "schemars 1.1.0", "serde", "serde_derive", "serde_json", @@ -11815,7 +11825,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12246,7 +12256,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12311,7 +12321,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12334,7 +12344,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.108", + "syn 2.0.110", "tokio", "url", ] @@ -12498,7 +12508,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12567,7 +12577,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12579,7 +12589,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12698,7 +12708,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12747,7 +12757,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12832,7 +12842,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12939,7 +12949,7 @@ checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12950,7 +12960,7 @@ checksum = "f486687bfb7b5c560868f69ed2d458b880cebc9babebcb67e49f31b55c5bf847" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12973,7 +12983,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -12989,9 +12999,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.108" +version = "2.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" +checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" dependencies = [ "proc-macro2", "quote", @@ -13033,7 +13043,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -13391,7 +13401,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -13402,7 +13412,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -13648,7 +13658,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -13819,9 +13829,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.16" +version = "0.7.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" dependencies = [ "bytes", "futures-core", @@ -13935,7 +13945,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -14078,7 +14088,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -14331,7 +14341,7 @@ checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -14814,7 +14824,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", "wasm-bindgen-shared", ] @@ -14849,7 +14859,7 @@ checksum = "ffc003a991398a8ee604a401e194b6b3a39677b3173d6e74495eb51b82e99a32" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -14884,7 +14894,7 @@ checksum = "a369369e4360c2884c3168d22bded735c43cccae97bbc147586d4b480edd138d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -14951,14 +14961,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" dependencies = [ - "webpki-root-certs 1.0.3", + "webpki-root-certs 1.0.4", ] [[package]] name = "webpki-root-certs" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d651ec480de84b762e7be71e6efa7461699c19d9e2c272c8d93455f567786e" +checksum = "ee3e3b5f5e80bc89f30ce8d0343bf4e5f12341c51f3e26cbeecbc7c85443e85b" dependencies = [ "rustls-pki-types", ] @@ -14969,14 +14979,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.3", + "webpki-roots 1.0.4", ] [[package]] name = "webpki-roots" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" dependencies = [ "rustls-pki-types", ] @@ -15138,7 +15148,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "aws-sdk-config", @@ -15198,7 +15208,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "argon2", @@ -15241,7 +15251,7 @@ dependencies = [ "hf-hub", "hmac", "http 1.3.1", - "hyper 1.7.0", + "hyper 1.8.0", "indexmap 2.11.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -15318,7 +15328,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.571.0" +version = "1.575.0" dependencies = [ "base64 0.22.1", "chrono", @@ -15333,7 +15343,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.571.0" +version = "1.575.0" dependencies = [ "chrono", "lazy_static", @@ -15347,7 +15357,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "axum", @@ -15366,7 +15376,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "async-recursion", @@ -15394,7 +15404,7 @@ dependencies = [ "globset", "hex", "hmac", - "hyper 1.7.0", + "hyper 1.8.0", "indexmap 2.11.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -15451,7 +15461,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.571.0" +version = "1.575.0" dependencies = [ "regex", "serde", @@ -15466,7 +15476,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "bytes", @@ -15490,19 +15500,19 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.571.0" +version = "1.575.0" dependencies = [ "itertools 0.14.0", "lazy_static", "proc-macro2", "quote", "regex", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] name = "windmill-parser" -version = "1.571.0" +version = "1.575.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15511,7 +15521,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "lazy_static", @@ -15523,7 +15533,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "serde_json", @@ -15535,7 +15545,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "gosyn", @@ -15547,7 +15557,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "lazy_static", @@ -15559,7 +15569,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "serde_json", @@ -15571,7 +15581,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "nu-parser", @@ -15582,7 +15592,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15593,7 +15603,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15605,7 +15615,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "async-recursion", @@ -15628,7 +15638,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "lazy_static", @@ -15642,7 +15652,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15652,14 +15662,14 @@ dependencies = [ "quote", "regex", "serde_json", - "syn 2.0.108", + "syn 2.0.110", "toml", "windmill-parser", ] [[package]] name = "windmill-parser-sql" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "lazy_static", @@ -15673,7 +15683,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "lazy_static", @@ -15691,7 +15701,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "serde", @@ -15702,7 +15712,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "async-recursion", @@ -15718,6 +15728,7 @@ dependencies = [ "itertools 0.14.0", "lazy_static", "prometheus", + "quick_cache", "regex", "reqwest 0.12.24", "serde", @@ -15725,6 +15736,7 @@ dependencies = [ "serde_urlencoded", "sql-builder", "sqlx", + "thiserror 2.0.17", "tokio", "tracing", "ulid", @@ -15735,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.571.0" +version = "1.575.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15745,7 +15757,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.571.0" +version = "1.575.0" dependencies = [ "anyhow", "async-once-cell", @@ -15776,6 +15788,7 @@ dependencies = [ "deno_webidl", "dotenv", "dyn-iter", + "eventsource-stream", "flume", "futures", "gcp_auth", @@ -15956,7 +15969,7 @@ checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -15967,7 +15980,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -15978,7 +15991,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -15989,7 +16002,7 @@ checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -16000,7 +16013,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -16011,7 +16024,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -16508,7 +16521,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", "synstructure 0.13.2", ] @@ -16520,7 +16533,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", "synstructure 0.13.2", ] @@ -16541,7 +16554,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -16561,7 +16574,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", "synstructure 0.13.2", ] @@ -16582,7 +16595,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] @@ -16615,7 +16628,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.110", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index c1ed86fc84..5702c3a8f5 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.571.0" +version = "1.575.0" authors.workspace = true edition.workspace = true @@ -33,7 +33,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.571.0" +version = "1.575.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -252,6 +252,7 @@ urlencoding = "^2" url = { version = "^2" , features = ["serde"]} async-oauth2 = "0.5.1" reqwest = { version = "^0.12", features = ["json", "stream", "gzip", "multipart"] } +eventsource-stream = "0.2.3" time = "^0" serde_urlencoded = "^0" tokio-tar = "^0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index dbf861025b..0da534da09 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6a3c688d51480b764d780bb02368c4b1da56caff +7cdb62d968feef22046b12a1217b711af457cced \ No newline at end of file diff --git a/backend/migrations/20251104135208_ai_agent_memory.down.sql b/backend/migrations/20251104135208_ai_agent_memory.down.sql new file mode 100644 index 0000000000..6b659bcadc --- /dev/null +++ b/backend/migrations/20251104135208_ai_agent_memory.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here + +-- Drop table +DROP TABLE IF EXISTS ai_agent_memory; diff --git a/backend/migrations/20251104135208_ai_agent_memory.up.sql b/backend/migrations/20251104135208_ai_agent_memory.up.sql new file mode 100644 index 0000000000..168659eb72 --- /dev/null +++ b/backend/migrations/20251104135208_ai_agent_memory.up.sql @@ -0,0 +1,16 @@ +-- Add up migration script here + +-- Create ai_agent_memory table for storing AI agent step memory when S3 is unavailable +CREATE TABLE ai_agent_memory ( + workspace_id VARCHAR(50) NOT NULL, + conversation_id UUID NOT NULL, + step_id VARCHAR(255) NOT NULL, + messages JSONB NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + PRIMARY KEY (workspace_id, conversation_id, step_id) +); + +-- Grant permissions +GRANT ALL ON ai_agent_memory TO windmill_admin; +GRANT ALL ON ai_agent_memory TO windmill_user; diff --git a/backend/migrations/20251105100125_legacy_sql_result_flag.down.sql b/backend/migrations/20251105100125_legacy_sql_result_flag.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20251105100125_legacy_sql_result_flag.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20251105100125_legacy_sql_result_flag.up.sql b/backend/migrations/20251105100125_legacy_sql_result_flag.up.sql new file mode 100644 index 0000000000..dbacc6084c --- /dev/null +++ b/backend/migrations/20251105100125_legacy_sql_result_flag.up.sql @@ -0,0 +1,99 @@ +CREATE OR REPLACE FUNCTION update_string(s text) +RETURNS text +LANGUAGE plpgsql +AS $$ +DECLARE + prefix TEXT := '-- https://www.windmill.dev/docs/getting_started/scripts_quickstart/sql#result-collection +-- result_collection=legacy + +'; +BEGIN + RETURN prefix || s; +END; +$$; + +CREATE OR REPLACE FUNCTION update_all_modules(obj jsonb) +RETURNS jsonb +LANGUAGE plpgsql +AS $$ +DECLARE + result jsonb; + k text; + v jsonb; +BEGIN + IF jsonb_typeof(obj) = 'object' THEN + result := '{}'::jsonb; + + FOR k, v IN SELECT * FROM jsonb_each(obj) + LOOP + IF k = 'content' and jsonb_typeof(v) = 'string' AND obj->>'language' IN ('bigquery', 'postgresql', 'duckdb', 'mssql', 'oracledb', 'snowflake', 'mysql') THEN + result := result || jsonb_build_object('content', update_string(obj->>'content')); + ELSE + result := result || jsonb_build_object(k, update_all_modules(v)); + END IF; + END LOOP; + + RETURN result; + ELSIF jsonb_typeof(obj) = 'array' AND jsonb_array_length(obj) > 0 THEN + SELECT jsonb_agg(update_all_modules(elem)) + INTO result + FROM jsonb_array_elements(obj) elem; + + RETURN result; + ELSE + RETURN obj; + END IF; +END; +$$; + +-- Run on a flow_version_lite jsonb value. Returns an array of flow_node ids whose languages are SQL. +CREATE OR REPLACE FUNCTION find_sql_flow_nodes_ids(obj jsonb) +RETURNS BIGINT[] +LANGUAGE plpgsql +AS $$ +DECLARE + result BIGINT[] := '{}'; + k text; + v jsonb; +BEGIN + IF jsonb_typeof(obj) = 'object' THEN + IF obj->>'language' IN ('bigquery', 'postgresql', 'duckdb', 'mssql', 'oracledb', 'snowflake', 'mysql') AND jsonb_typeof(obj->'id') = 'number' THEN + result := result || (obj->>'id')::BIGINT; + END IF; + + FOR k, v IN SELECT * FROM jsonb_each(obj) + LOOP + result := result || find_sql_flow_nodes_ids(v); + END LOOP; + ELSIF jsonb_typeof(obj) = 'array' AND jsonb_array_length(obj) > 0 THEN + SELECT array_agg(result_ids) + INTO result + FROM jsonb_array_elements(obj) elem, unnest(find_sql_flow_nodes_ids(elem)) as result_ids; + END IF; + RETURN result; +END; +$$; + +DO $$ +BEGIN + UPDATE app_version SET value = update_all_modules(value::jsonb)::json; + UPDATE draft SET value = update_all_modules(value::jsonb)::json; + UPDATE flow SET value = update_all_modules(value); + UPDATE flow_version SET value = update_all_modules(value); + UPDATE flow_node SET code = update_string(code) WHERE id IN ( + SELECT v FROM flow_version_lite, unnest(find_sql_flow_nodes_ids(value)) as v + ); + UPDATE app_script SET code = update_string(code) WHERE id IN ( + SELECT v FROM app_version_lite, unnest(find_sql_flow_nodes_ids(value)) as v + ); + UPDATE script SET content = update_string(content) WHERE language IN ('bigquery', 'postgresql', 'duckdb', 'mssql', 'oracledb', 'snowflake', 'mysql'); + +EXCEPTION WHEN OTHERS THEN + -- ✅ LOG ERROR WITHOUT STOPPING THE MIGRATION + RAISE WARNING 'Migration failed: %', SQLERRM; +END; +$$; + +DROP FUNCTION IF EXISTS update_all_modules(jsonb); +DROP FUNCTION IF EXISTS update_string(text); +DROP FUNCTION IF EXISTS find_sql_flow_nodes_ids(jsonb); diff --git a/backend/parsers/windmill-parser-bash/src/lib.rs b/backend/parsers/windmill-parser-bash/src/lib.rs index 4e3d70c038..5bfe6b20c5 100644 --- a/backend/parsers/windmill-parser-bash/src/lib.rs +++ b/backend/parsers/windmill-parser-bash/src/lib.rs @@ -47,7 +47,6 @@ pub fn parse_powershell_sig(code: &str) -> anyhow::Result { lazy_static::lazy_static! { static ref RE_BASH: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+)\}|\{(\d+):-(.*)\})"(?:[\t ]*)?(?:#.*)?\r?$"#).unwrap(); - pub static ref RE_POWERSHELL_PARAM: Regex = Regex::new(r#"(?m)param[\t ]*\(([^)]*)\)"#).unwrap(); static ref RE_POWERSHELL_ARGS: Regex = Regex::new(r#"(?:\[([\w\[\]]+)\])?\$(\w+)[\t ]*(?:=[\t ]*(?:(?:(?:"|')([^"\n\r\$]*)(?:"|'))|([\d.]+)))?\r?"#).unwrap(); } @@ -85,6 +84,136 @@ fn parse_bash_file(code: &str) -> anyhow::Result>> { Ok(Some(args)) } +/// Extract a PowerShell param() block, handling nested parentheses. +/// +/// # Arguments +/// * `code` - The PowerShell code to extract from +/// * `include_keyword` - If true, returns the full block including "param(...)" +/// If false, returns only the contents between the parentheses +/// +/// # Returns +/// The extracted param block or contents, or None if not found. +pub fn extract_powershell_param_block(code: &str, include_keyword: bool) -> Option<&str> { + // Find "param" keyword (case-insensitive) + let lower_code = code.to_lowercase(); + let param_start = lower_code.find("param")?; + + // Verify that only comments and whitespace appear before "param" + let before_param = &code[..param_start]; + let mut chars = before_param.chars().peekable(); + let mut in_block_comment = false; + + while let Some(ch) = chars.next() { + if in_block_comment { + // Check for end of block comment: #> + if ch == '#' && chars.peek() == Some(&'>') { + chars.next(); // consume '>' + in_block_comment = false; + } + } else { + match ch { + // Start of block comment: <# + '<' if chars.peek() == Some(&'#') => { + chars.next(); // consume '#' + in_block_comment = true; + } + // Single-line comment: consume until end of line + '#' => { + while let Some(&next_ch) = chars.peek() { + if next_ch == '\n' || next_ch == '\r' { + break; + } + chars.next(); + } + } + // Whitespace is allowed + c if c.is_whitespace() => {} + // Any other character means there's code before param + _ => return None, + } + } + } + + // If we're still in a block comment at the end, it's unclosed - invalid + if in_block_comment { + return None; + } + + // Skip whitespace and tabs after "param" + let mut chars = code[param_start + 5..].char_indices(); + let mut paren_offset = param_start + 5; + + // Skip whitespace to find opening paren + while let Some((idx, ch)) = chars.next() { + if ch == '(' { + paren_offset += idx; + break; + } else if !ch.is_whitespace() && ch != '\t' { + // Found non-whitespace, non-paren character - not a valid param block + return None; + } + } + + // Now parse from the opening parenthesis + let remaining = &code[paren_offset..]; + let mut chars = remaining.char_indices(); + + // Skip the opening '(' + if let Some((_, ch)) = chars.next() { + if ch != '(' { + return None; + } + } else { + return None; + } + + let mut depth = 1; + let mut in_single_quote = false; + let mut in_double_quote = false; + let mut escape_next = false; + let content_start = paren_offset + 1; // Start after the opening '(' + + for (idx, ch) in chars { + if escape_next { + escape_next = false; + continue; + } + + match ch { + '`' if in_double_quote => { + // PowerShell escape character + escape_next = true; + } + '\'' if !in_double_quote => { + in_single_quote = !in_single_quote; + } + '"' if !in_single_quote => { + in_double_quote = !in_double_quote; + } + '(' if !in_single_quote && !in_double_quote => { + depth += 1; + } + ')' if !in_single_quote && !in_double_quote => { + depth -= 1; + if depth == 0 { + // Found the matching closing parenthesis + // idx is the position of ')' relative to paren_offset + if include_keyword { + // Return full block including "param" keyword and closing paren + return Some(&code[param_start..paren_offset + idx + 1]); + } else { + // Return only contents between parentheses + return Some(&code[content_start..paren_offset + idx]); + } + } + } + _ => {} + } + } + + None +} + enum ParserState { Normal, InSingleQuote, @@ -137,10 +266,9 @@ fn parse_powershell_single_typ(typ: &str) -> Typ { } fn parse_powershell_file(code: &str) -> anyhow::Result>> { - let param_wrapper = RE_POWERSHELL_PARAM.captures(code); + let param_wrapper = extract_powershell_param_block(code, false); let mut args = vec![]; if let Some(param_wrapper) = param_wrapper { - let param_wrapper = param_wrapper.get(1).unwrap().as_str(); let params = split_pwsh_args(param_wrapper); for param in params { if let Some(cap) = RE_POWERSHELL_ARGS.captures(param) { @@ -266,7 +394,7 @@ non_required="${5:-}" #[test] fn test_parse_powershell_sig() -> anyhow::Result<()> { - let code = r#"param($Msg, [string]$Msg2, $Dflt = "default value, with comma", [int]$Nb = 3 , $Nb2 = 5.0, $Nb3 = 5, $Wahoo = $env:WAHOO, [PSCustomObject]$Obj, [string[]]$Arr)"#; + let code = r#"param($Msg, [string]$Msg2, $Dflt = "default value, with comma", [int]$Nb = 3 , $Nb2 = 5.0, $Nb3 = 5, $Wahoo = $env:WAHOO, [PSCustomObject]$Obj, [string[]]$Arr, [Parameter(Mandatory)][ValidateSet('Green', 'Blue', 'Red')][string]$Message)"#; assert_eq!( parse_powershell_sig(code)?, MainArgSignature { @@ -344,6 +472,14 @@ non_required="${5:-}" default: None, has_default: false, oidx: None + }, + Arg { + otyp: None, + name: "Message".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None } ], no_main_func: None, @@ -353,6 +489,121 @@ non_required="${5:-}" Ok(()) } + #[test] + fn test_extract_powershell_param_block() { + // Basic cases + assert_eq!( + extract_powershell_param_block("param($Name, $Age)", true), + Some("param($Name, $Age)") + ); + assert_eq!( + extract_powershell_param_block("param($Name, $Age)", false), + Some("$Name, $Age") + ); + + // Case insensitive and whitespace + assert_eq!( + extract_powershell_param_block("PARAM ($Value)", false), + Some("$Value") + ); + + // Nested parentheses + assert_eq!( + extract_powershell_param_block("param([ValidateScript({$_ -gt 0})]$Count)", false), + Some("[ValidateScript({$_ -gt 0})]$Count") + ); + + // Strings with parentheses + assert_eq!( + extract_powershell_param_block("param([string]$Path = 'C:\\file(1).txt')", false), + Some("[string]$Path = 'C:\\file(1).txt'") + ); + assert_eq!( + extract_powershell_param_block(r#"param($Msg = "Hello (world)")"#, false), + Some(r#"$Msg = "Hello (world)""#) + ); + assert_eq!( + extract_powershell_param_block( + r#"param([Parameter(Mandatory)][ValidateSet('Green', 'Blue', 'Red')][string]$Message)"#, + false + ), + Some(r#"[Parameter(Mandatory)][ValidateSet('Green', 'Blue', 'Red')][string]$Message"#) + ); + + // Escaped quotes + assert_eq!( + extract_powershell_param_block("param($Text = 'don''t')", false), + Some("$Text = 'don''t'") + ); + assert_eq!( + extract_powershell_param_block(r#"param($Text = "He said `"Hi`"")"#, false), + Some(r#"$Text = "He said `"Hi`"""#) + ); + + // Multiline + let multiline = "param(\n [string]$Name,\n [int]$Age\n)"; + assert!(extract_powershell_param_block(multiline, false).is_some()); + + // Invalid cases + assert_eq!(extract_powershell_param_block("$x = 5", false), None); + assert_eq!(extract_powershell_param_block("param", false), None); + assert_eq!(extract_powershell_param_block("param($x", false), None); + + // Valid: param at beginning with single-line comments before + assert_eq!( + extract_powershell_param_block("# This is a comment\nparam($Name)", false), + Some("$Name") + ); + assert_eq!( + extract_powershell_param_block("# Comment 1\n# Comment 2\n\nparam($Name)", false), + Some("$Name") + ); + + // Valid: param at beginning with block comment before + assert_eq!( + extract_powershell_param_block("<# Block comment #>\nparam($Name)", false), + Some("$Name") + ); + assert_eq!( + extract_powershell_param_block( + "<#\n Multi-line\n block comment\n#>\nparam($Name)", + false + ), + Some("$Name") + ); + + // Valid: mixed comments and whitespace + assert_eq!( + extract_powershell_param_block( + "# Line comment\n<# Block comment #>\n\nparam($Name)", + false + ), + Some("$Name") + ); + + // Invalid: code before param + assert_eq!( + extract_powershell_param_block("$x = 5\nparam($Name)", false), + None + ); + assert_eq!( + extract_powershell_param_block("Write-Host 'test'\nparam($Name)", false), + None + ); + + // Invalid: unclosed block comment + assert_eq!( + extract_powershell_param_block("<# Unclosed comment\nparam($Name)", false), + None + ); + + // Invalid: unclosed block comment + assert_eq!( + extract_powershell_param_block("function test-x{ param($Name)\n}", false), + None + ); + } + #[test] fn test_parse_bash_sig_with_crlf() -> anyhow::Result<()> { // Test with CRLF line endings (Windows-style) diff --git a/backend/parsers/windmill-parser-php/src/lib.rs b/backend/parsers/windmill-parser-php/src/lib.rs index 1ac2b9a63a..14f30f940c 100644 --- a/backend/parsers/windmill-parser-php/src/lib.rs +++ b/backend/parsers/windmill-parser-php/src/lib.rs @@ -44,23 +44,41 @@ fn parse_default_expr(e: Expression) -> Option { pub fn parse_php_signature( code: &str, - override_main: Option, + override_entrypoint: Option, ) -> anyhow::Result { - let main_name = override_main.unwrap_or("main".to_string()); + let entrypoint_fn_name = override_entrypoint.unwrap_or("main".to_string()); let ast = parser::parse(code) .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; - let params = ast.into_iter().find_map(|x| match x { - Statement::Function(FunctionStatement { - name, - parameters: FunctionParameterList { parameters, .. }, - .. - }) if name.to_string() == main_name => Some(parameters), - _ => None, - }); + let mut entrypoint_params = None; + let mut has_preprocessor = None; + for node in ast.into_iter() { + match node { + Statement::Function(FunctionStatement { + name, + parameters: FunctionParameterList { parameters, .. }, + .. + }) => { + let fn_name = name.to_string(); - if let Some(params) = params { + if has_preprocessor.is_none() && fn_name == "preprocessor" { + has_preprocessor = Some(true); + } + + if entrypoint_params.is_none() && fn_name == entrypoint_fn_name { + entrypoint_params = Some(parameters); + } + + if has_preprocessor.is_some() && entrypoint_params.is_some() { + break; + } + } + _ => {} + }; + } + + if let Some(params) = entrypoint_params { let args = params .into_iter() .map(|x| { @@ -82,7 +100,7 @@ pub fn parse_php_signature( star_kwargs: false, args, no_main_func: Some(false), - has_preprocessor: None, + has_preprocessor, }) } else { Ok(MainArgSignature { @@ -90,7 +108,7 @@ pub fn parse_php_signature( star_kwargs: false, args: vec![], no_main_func: Some(true), - has_preprocessor: None, + has_preprocessor, }) } } diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index f3b7929d38..f6de0a7b05 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -134,8 +134,11 @@ pub fn parse_graphql(code: &str) -> String { #[cfg(feature = "php-parser")] #[wasm_bindgen] -pub fn parse_php(code: &str) -> String { - wrap_sig(windmill_parser_php::parse_php_signature(code, None)) +pub fn parse_php(code: &str, main_override: Option) -> String { + wrap_sig(windmill_parser_php::parse_php_signature( + code, + main_override, + )) } #[cfg(feature = "rust-parser")] diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index fdd5cbd148..b0818c96cd 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -59,7 +59,6 @@ use windmill_common::{ SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, indexer::load_indexer_config, - jobs::QueuedJob, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, @@ -80,7 +79,7 @@ use windmill_common::{ OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, }; use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING}; -use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload}; +use windmill_queue::{SameWorkerPayload, cancel_job, get_queued_job_v2}; use windmill_worker::{ handle_job_error, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, @@ -2047,7 +2046,7 @@ async fn cancel_stale_job( const RESTART_LIMIT: i32 = 3; -async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker_name: &str) { +async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_name: &str) { let mut zombie_jobs_uuid_restart_limit_reached = vec![]; if *RESTART_ZOMBIE_JOBS { @@ -2220,24 +2219,18 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker ); } - let jobs = sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status FROM v2_as_queue WHERE id = ANY($1)", - ) - .bind(&timeouts[..]) - .fetch_all(db) - .await - .map_err(|e| tracing::error!("Error fetching same worker jobs: {:?}", e)) - .unwrap_or_default(); - jobs + timeouts }; let non_restartable_jobs = if *RESTART_ZOMBIE_JOBS { vec![] } else { - sqlx::query_as::<_, QueuedJob>("SELECT *, null as workflow_as_code_status FROM v2_as_queue WHERE last_ping < now() - ($1 || ' seconds')::interval - AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND same_worker = false") - .bind(ZOMBIE_JOB_TIMEOUT.as_str()) + sqlx::query_scalar!("SELECT j.id + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE r.ping < now() - ($1 || ' seconds')::interval + AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false", + ZOMBIE_JOB_TIMEOUT.as_str()) .fetch_all(db) .await .ok() @@ -2260,14 +2253,6 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker } } - let zombie_jobs_restart_limit_reached = sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status FROM v2_as_queue WHERE id = ANY($1)", - ) - .bind(&zombie_jobs_uuid_restart_limit_reached[..]) - .fetch_all(db) - .await - .ok() - .unwrap_or_else(|| vec![]); let timeouts = non_restartable_jobs .into_iter() @@ -2278,7 +2263,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker .map(|x| (x, ErrorMessage::SameWorker)), ) .chain( - zombie_jobs_restart_limit_reached + zombie_jobs_uuid_restart_limit_reached .into_iter() .map(|x| (x, ErrorMessage::RestartLimit)), ) @@ -2289,7 +2274,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _); } - for (job, error_kind) in timeouts { + for (job_id, error_kind) in timeouts { // since the job is unrecoverable, the same worker queue should never be sent anything let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::(1); @@ -2298,6 +2283,12 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker let (send_result_never_used, _send_result_rx_never_used) = JobCompletedSender::new_never_used(); + let job = get_queued_job_v2(db, &job_id).await; + if let Err(e) = job { + tracing::error!("Error getting queued job: {:?}", e); + continue; + } + if let Some(job) = job.unwrap() { let label = if job.permissioned_as != format!("u/{}", job.created_by) && job.permissioned_as != job.created_by { @@ -2311,7 +2302,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker &job.permissioned_as, &label, *SCRIPT_TOKEN_EXPIRY, - &job.email, + &job.permissioned_as_email, &job.id, None, Some(format!("handle_zombie_jobs")), @@ -2326,32 +2317,32 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker None, ); - let last_ping = job.last_ping.clone(); let error_message = format!( - "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, reason: {:?})", - last_ping - .map(|x| x.to_string()) - .unwrap_or_else(|| "no ping".to_string()), + "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, reason: {:?}).\nThis likely means that the job died on worker {}, OOM are a common reason for worker crashes.\nCheck the workers around the time of the last ping and the exit code if any.", + job.last_ping.unwrap_or_default(), *ZOMBIE_JOB_TIMEOUT, - error_kind.to_string() + error_kind.to_string(), + job.worker.clone().unwrap_or_default(), ); + let memory_peak = job.memory_peak.unwrap_or(0); let _ = handle_job_error( db, &client, - &MiniPulledJob::from(&job), - 0, + &windmill_queue::MiniCompletedJob::from(job), + memory_peak, None, error::Error::ExecutionErr(error_message), true, Some(&same_worker_tx_never_used), "", - worker_name, + node_name, send_result_never_used, #[cfg(feature = "benchmark")] &mut windmill_common::bench::BenchmarkIter::new(), ) .await; } + } } async fn cleanup_concurrency_counters_orphaned_keys(db: &DB) -> error::Result<()> { @@ -2457,13 +2448,13 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { let flows = sqlx::query!( r#" SELECT - id AS "id!", workspace_id AS "workspace_id!", parent_job, is_flow_step, - flow_status AS "flow_status: Box", last_ping, same_worker - FROM v2_as_queue - WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now() - AND (job_kind = 'flow' OR job_kind = 'flowpreview' OR job_kind = 'flownode') - AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval - AND canceled = false + j.id AS "id!", j.workspace_id AS "workspace_id!", j.parent_job, j.flow_step_id IS NOT NULL AS "is_flow_step?", + COALESCE(s.flow_status, s.workflow_as_code_status) AS "flow_status: Box", r.ping AS last_ping, j.same_worker AS "same_worker?" + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now() + AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode') + AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval + AND q.canceled_by IS NULL "#, FLOW_ZOMBIE_TRANSITION_TIMEOUT.as_str() diff --git a/backend/substitute_ee_code.sh b/backend/substitute_ee_code.sh index 3ea73e16cc..7cd9a0b25a 100755 --- a/backend/substitute_ee_code.sh +++ b/backend/substitute_ee_code.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" root_dirpath="$(cd "${script_dirpath}/.." && pwd)" diff --git a/backend/tests/common/mod.rs b/backend/tests/common/mod.rs index 475a8540b7..40a9980a96 100644 --- a/backend/tests/common/mod.rs +++ b/backend/tests/common/mod.rs @@ -394,7 +394,17 @@ pub async fn listen_for_uuid_on( pub async fn completed_job(uuid: Uuid, db: &Pool) -> CompletedJob { sqlx::query_as::<_, CompletedJob>( - "SELECT *, result->'wm_labels' as labels FROM v2_as_completed_job WHERE id = $1", + "SELECT j.id, j.workspace_id, j.parent_job, j.created_by, j.created_at, c.duration_ms, + c.status = 'success' OR c.status = 'skipped' AS success, j.runnable_id AS script_hash, j.runnable_path AS script_path, + j.args, c.result, FALSE AS deleted, j.raw_code, c.status = 'canceled' AS canceled, + c.canceled_by, c.canceled_reason, j.kind AS job_kind, + CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END AS schedule_path, + j.permissioned_as, COALESCE(c.flow_status, c.workflow_as_code_status) AS flow_status, j.raw_flow, + j.flow_step_id IS NOT NULL AS is_flow_step, j.script_lang AS language, c.started_at, + c.status = 'skipped' AS is_skipped, j.raw_lock, j.permissioned_as_email AS email, j.visible_to_owner, + c.memory_peak AS mem_peak, j.tag, j.priority, NULL::TEXT AS logs, c.result_columns, + j.script_entrypoint_override, j.preprocessed, c.result->'wm_labels' as labels + FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1", ) .bind(uuid) .fetch_one(db) diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 1129846bc9..72b6aeea52 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -2395,7 +2395,7 @@ async fn test_script_schedule_handlers(db: Pool) -> anyhow::Result<()> let uuid = uuid.unwrap().unwrap(); let completed_job = sqlx::query!( - "SELECT script_path FROM v2_as_completed_job WHERE id = $1", + "SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid ) .fetch_one(&db2) @@ -2466,7 +2466,7 @@ async fn test_script_schedule_handlers(db: Pool) -> anyhow::Result<()> let uuid = uuid.unwrap().unwrap(); let completed_job = - sqlx::query!("SELECT script_path FROM v2_as_completed_job WHERE id = $1", uuid) + sqlx::query!("SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid) .fetch_one(&db2) .await .unwrap(); @@ -2553,7 +2553,7 @@ async fn test_flow_schedule_handlers(db: Pool) -> anyhow::Result<()> { let uuid = uuid.unwrap().unwrap(); let completed_job = sqlx::query!( - "SELECT script_path FROM v2_as_completed_job WHERE id = $1", + "SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid ) .fetch_one(&db2) @@ -2625,7 +2625,7 @@ async fn test_flow_schedule_handlers(db: Pool) -> anyhow::Result<()> { let uuid = uuid.unwrap().unwrap(); let completed_job = - sqlx::query!("SELECT script_path FROM v2_as_completed_job WHERE id = $1", uuid) + sqlx::query!("SELECT runnable_path as script_path FROM v2_job WHERE id = $1", uuid) .fetch_one(&db2) .await .unwrap(); diff --git a/backend/update_sqlx.sh b/backend/update_sqlx.sh index 770bc760cb..5425706876 100755 --- a/backend/update_sqlx.sh +++ b/backend/update_sqlx.sh @@ -2,9 +2,6 @@ set -e -# Default directory -EE_DIR="../windmill-ee-private" - # Parse arguments while [[ "$#" -gt 0 ]]; do case $1 in @@ -14,8 +11,6 @@ while [[ "$#" -gt 0 ]]; do shift done -./substitute_ee_code.sh --dir "$EE_DIR" - # Check if running on macOS if [[ "$(uname)" == "Darwin" ]]; then echo "Running on macOS - substituting samael..." diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 034c7d270e..eecf52efbf 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.571.0 + version: 1.575.0 title: Windmill API contact: @@ -629,12 +629,13 @@ paths: tags: - Git Sync parameters: - - name: search + - name: page in: query - description: Search repositories by name + description: Page number for pagination (default 1) required: false schema: - type: string + type: integer + default: 1 responses: "200": description: connected repositories @@ -727,6 +728,32 @@ paths: schema: type: string + /w/{workspace}/workspaces/create_workspace_fork_branch: + post: + summary: create forked workspace branch with git sync + operationId: createWorkspaceForkGitBranch + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: new forked workspace + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateWorkspaceFork" + responses: + "201": + description: forked workspace branch created + content: + application/json: + schema: + type: array + items: + type: string + format: uuid + /w/{workspace}/workspaces/create_fork: post: summary: create forked workspace @@ -4871,6 +4898,27 @@ paths: - content - language + /scripts/hub/pick/{path}: + get: + summary: record hub script pick + operationId: pickHubScriptByPath + tags: + - script + parameters: + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: script pick recorded + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + required: + - success + /scripts/hub/top: get: summary: get top hub scripts @@ -7946,6 +7994,12 @@ paths: schema: type: boolean - $ref: "#/components/parameters/NewJobId" + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid requestBody: description: preview @@ -7972,6 +8026,12 @@ paths: - job parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid requestBody: description: preview diff --git a/backend/windmill-api/src/approvals.rs b/backend/windmill-api/src/approvals.rs index e03b587f10..1296000481 100644 --- a/backend/windmill-api/src/approvals.rs +++ b/backend/windmill-api/src/approvals.rs @@ -204,31 +204,31 @@ pub async fn get_approval_form_details( "WITH job_info AS ( -- Query for Teams (running jobs) SELECT - parent.job_kind AS \"job_kind!: JobKind\", - parent.script_hash AS \"script_hash: ScriptHash\", - parent.raw_flow AS \"raw_flow: sqlx::types::Json>\", - child.parent_job AS \"parent_job: Uuid\", - parent.created_at AS \"created_at!: chrono::NaiveDateTime\", - parent.created_by AS \"created_by!\", - parent.script_path, - parent.args AS \"args: sqlx::types::Json>\" - FROM v2_as_queue child - JOIN v2_as_queue parent ON parent.id = child.parent_job - WHERE child.id = $1 AND child.workspace_id = $2 + parent_j.kind AS \"job_kind!: JobKind\", + parent_j.runnable_id AS \"script_hash: ScriptHash\", + parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\", + child_j.parent_job AS \"parent_job: Uuid\", + parent_j.created_at AS \"created_at!: chrono::NaiveDateTime\", + parent_j.created_by AS \"created_by!\", + parent_j.runnable_path as script_path, + parent_j.args AS \"args: sqlx::types::Json>\" + FROM v2_job_queue child_q JOIN v2_job child_j USING (id) + JOIN v2_job parent_j ON parent_j.id = child_j.parent_job + WHERE child_j.id = $1 AND child_j.workspace_id = $2 UNION ALL -- Query for Slack (completed jobs) SELECT - v2_as_queue.job_kind AS \"job_kind!: JobKind\", - v2_as_queue.script_hash AS \"script_hash: ScriptHash\", - v2_as_queue.raw_flow AS \"raw_flow: sqlx::types::Json>\", - v2_as_completed_job.parent_job AS \"parent_job: Uuid\", - v2_as_completed_job.created_at AS \"created_at!: chrono::NaiveDateTime\", - v2_as_completed_job.created_by AS \"created_by!\", - v2_as_queue.script_path, - v2_as_queue.args AS \"args: sqlx::types::Json>\" - FROM v2_as_queue - JOIN v2_as_completed_job ON v2_as_completed_job.parent_job = v2_as_queue.id - WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 + parent_j.kind AS \"job_kind!: JobKind\", + parent_j.runnable_id AS \"script_hash: ScriptHash\", + parent_j.raw_flow AS \"raw_flow: sqlx::types::Json>\", + completed_j.parent_job AS \"parent_job: Uuid\", + completed_j.created_at AS \"created_at!: chrono::NaiveDateTime\", + completed_j.created_by AS \"created_by!\", + parent_j.runnable_path as script_path, + parent_j.args AS \"args: sqlx::types::Json>\" + FROM v2_job_completed completed_c JOIN v2_job completed_j USING (id) + JOIN v2_job parent_j ON parent_j.id = completed_j.parent_job + WHERE completed_j.id = $1 AND completed_j.workspace_id = $2 ) SELECT * FROM job_info LIMIT 1", job_id, diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 1cd0750919..b669f5e9a9 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -2530,13 +2530,13 @@ async fn check_if_allowed_to_access_s3_file_from_app( || { sqlx::query_scalar!( r#"SELECT EXISTS ( - SELECT 1 FROM v2_as_completed_job - WHERE workspace_id = $2 - AND (job_kind = 'appscript' OR job_kind = 'preview') - AND created_by = 'anonymous' - AND started_at > now() - interval '3 hours' - AND script_path LIKE $3 || '/%' - AND result @> ('{"s3":"' || $1 || '"}')::jsonb + SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE j.workspace_id = $2 + AND (j.kind = 'appscript' OR j.kind = 'preview') + AND j.created_by = 'anonymous' + AND c.started_at > now() - interval '3 hours' + AND j.runnable_path LIKE $3 || '/%' + AND c.result @> ('{"s3":"' || $1 || '"}')::jsonb )"#, file_query.s3, w_id, diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs index 9b3d21844a..875bcf683a 100644 --- a/backend/windmill-api/src/embeddings.rs +++ b/backend/windmill-api/src/embeddings.rs @@ -441,7 +441,7 @@ impl EmbeddingsDb { &query_embedding, limit.unwrap_or(10) as usize, Some(&filter), - Some(0.75), + Some(0.8), ); let results: Result> = results @@ -480,7 +480,17 @@ impl EmbeddingsDb { }) .collect(); - results + let mut results = results?; + + if results.len() > 1 { + let top_score = results[0].score; + results = results + .into_iter() + .take_while(|r| (top_score - r.score) / top_score <= 0.05) + .collect(); + } + + Ok(results) } pub async fn query_resource_types( diff --git a/backend/windmill-api/src/flow_conversations.rs b/backend/windmill-api/src/flow_conversations.rs index 6f046c4738..0cbdb7aa55 100644 --- a/backend/windmill-api/src/flow_conversations.rs +++ b/backend/windmill-api/src/flow_conversations.rs @@ -11,7 +11,7 @@ use uuid::Uuid; use crate::db::ApiAuthed; use windmill_common::{ - db::UserDB, + db::{UserDB, DB}, error::{JsonResult, Result}, flow_conversations::MessageType, utils::{not_found_if_none, paginate, Pagination}, @@ -154,6 +154,7 @@ pub async fn get_or_create_conversation_with_id( async fn delete_conversation( authed: ApiAuthed, Extension(user_db): Extension, + Extension(db): Extension, Path((w_id, conversation_id)): Path<(String, Uuid)>, ) -> Result { let mut tx = user_db.clone().begin(&authed).await?; @@ -185,10 +186,14 @@ async fn delete_conversation( // Delete associated memory in background (non-blocking cleanup) let w_id_clone = w_id.clone(); + let db_clone = db.clone(); tokio::spawn(async move { - if let Err(e) = - windmill_worker::memory_oss::delete_conversation_memory(&w_id_clone, conversation_id) - .await + if let Err(e) = windmill_worker::memory_oss::delete_conversation_memory( + &db_clone, + &w_id_clone, + conversation_id, + ) + .await { tracing::error!( "Failed to delete memory for conversation {} in workspace {}: {:?}", diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 13488ba220..ee0114c621 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -1634,6 +1634,7 @@ mod tests { early_return: None, concurrency_key: None, chat_input_enabled: None, + flow_env: None, debounce_key: None, debounce_delay_s: None, ui: None, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 4c11515291..d695a6091d 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -281,6 +281,10 @@ pub fn workspaced_service() -> Router { "/result_by_id/:job_id/:node_id", get(get_result_by_id).layer(cors.clone()), ) + .route( + "/flow_env_by_flow_job_id/:flow_job_id/:var_name", + get(get_flow_env_by_flow_job_id).layer(cors.clone()), + ) .route("/run/dependencies", post(run_dependencies_job)) .route("/run/flow_dependencies", post(run_flow_dependencies_job)) .route( @@ -376,6 +380,59 @@ async fn get_root_job( Ok(Json(res)) } +async fn get_flow_env_by_flow_job_id( + authed: ApiAuthed, + tokened: Tokened, + Extension(db): Extension, + Path((w_id, flow_job_id, var_name)): Path<(String, Uuid, String)>, + Query(JsonPath { json_path, .. }): Query, +) -> windmill_common::error::JsonResult> { + let flow_env = sqlx::query_scalar!( + r#" + SELECT + CASE + WHEN flow_version.id IS NOT NULL THEN + (flow_version.value -> 'flow_env' -> $3) #> $4 + ELSE + (root_job.raw_flow -> 'flow_env' -> $3) #> $4 + END AS "flow_env: sqlx::types::Json>" + FROM + v2_job current_job + JOIN + v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id) + AND root_job.workspace_id = current_job.workspace_id + LEFT JOIN + flow_version ON flow_version.id = root_job.runnable_id + AND flow_version.path = root_job.runnable_path + AND flow_version.workspace_id = root_job.workspace_id + WHERE + current_job.id = $1 AND + current_job.workspace_id = $2"#, + flow_job_id, + w_id, + var_name, + json_path + .as_ref() + .map(|x| x.split(".").collect::>()) + .unwrap_or_default() as Vec<&str>, + ) + .fetch_optional(&db) + .await? + .map(|r| r.map(|x| x.0)) + .flatten() + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + + log_job_view( + &db, + Some(&authed), + Some(&tokened.token), + &w_id, + &flow_job_id, + ) + .await?; + Ok(Json(flow_env)) +} + async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result { let root_job = sqlx::query_scalar!( r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, @@ -841,101 +898,6 @@ macro_rules! get_job_query { } } -// CREATE OR REPLACE VIEW v2_as_queue AS -// SELECT -// j.id, -// j.workspace_id, -// j.parent_job, -// j.created_by, -// j.created_at, -// q.started_at, -// q.scheduled_for, -// q.running, -// j.runnable_id AS script_hash, -// j.runnable_path AS script_path, -// j.args, -// j.raw_code, -// q.canceled_by IS NOT NULL AS canceled, -// q.canceled_by, -// q.canceled_reason, -// r.ping AS last_ping, -// j.kind AS job_kind, -// CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END -// AS schedule_path, -// j.permissioned_as, -// COALESCE(s.flow_status, s.workflow_as_code_status) AS flow_status, -// j.raw_flow, -// j.flow_step_id IS NOT NULL AS is_flow_step, -// j.script_lang AS language, -// q.suspend, -// q.suspend_until, -// j.same_worker, -// j.raw_lock, -// j.pre_run_error, -// j.permissioned_as_email AS email, -// j.visible_to_owner, -// r.memory_peak AS mem_peak, -// j.flow_innermost_root_job AS root_job, -// s.flow_leaf_jobs AS leaf_jobs, -// j.tag, -// j.concurrent_limit, -// j.concurrency_time_window_s, -// j.timeout, -// j.flow_step_id, -// j.cache_ttl, -// j.priority, -// NULL::TEXT AS logs, -// j.script_entrypoint_override, -// j.preprocessed -// FROM v2_job_queue q -// JOIN v2_job j USING (id) -// LEFT JOIN v2_job_runtime r USING (id) -// LEFT JOIN v2_job_status s USING (id) -// ; - -// -- Add up migration script here -// CREATE OR REPLACE VIEW v2_as_completed_job AS -// SELECT -// j.id, -// j.workspace_id, -// j.parent_job, -// j.created_by, -// j.created_at, -// c.duration_ms, -// c.status = 'success' OR c.status = 'skipped' AS success, -// j.runnable_id AS script_hash, -// j.runnable_path AS script_path, -// j.args, -// c.result, -// FALSE AS deleted, -// j.raw_code, -// c.status = 'canceled' AS canceled, -// c.canceled_by, -// c.canceled_reason, -// j.kind AS job_kind, -// CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END -// AS schedule_path, -// j.permissioned_as, -// COALESCE(c.flow_status, c.workflow_as_code_status) AS flow_status, -// j.raw_flow, -// j.flow_step_id IS NOT NULL AS is_flow_step, -// j.script_lang AS language, -// c.started_at, -// c.status = 'skipped' AS is_skipped, -// j.raw_lock, -// j.permissioned_as_email AS email, -// j.visible_to_owner, -// c.memory_peak AS mem_peak, -// j.tag, -// j.priority, -// NULL::TEXT AS logs, -// c.result_columns, -// j.script_entrypoint_override, -// j.preprocessed -// FROM v2_job_completed c -// JOIN v2_job j USING (id) -// ; - #[derive(Copy, Clone)] struct GetQuery<'a> { with_logs: bool, @@ -1506,10 +1468,10 @@ async fn get_job_logs( .flatten(); let record = sqlx::query!( - "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index - FROM v2_as_completed_job - LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id - WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_completed_job.tag = ANY($3))", + "SELECT j.created_by AS \"created_by\", coalesce(job_logs.logs, '') as logs, COALESCE(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index + FROM v2_job j + LEFT JOIN job_logs ON job_logs.job_id = j.id + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", id, w_id, tags.as_ref().map(|v| v.as_slice()) @@ -1535,11 +1497,21 @@ async fn get_job_logs( .await?; #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(r) = get_logs_from_store(record.log_offset, &logs, &record.log_file_index).await + if let Some(r) = get_logs_from_store( + record.log_offset.unwrap_or(0), + &logs, + &record.log_file_index, + ) + .await { return r.map(content_plain); } - if let Some(r) = get_logs_from_disk(record.log_offset, &logs, &record.log_file_index).await + if let Some(r) = get_logs_from_disk( + record.log_offset.unwrap_or(0), + &logs, + &record.log_file_index, + ) + .await { return r.map(content_plain); } @@ -1554,10 +1526,10 @@ async fn get_job_logs( Ok(content_plain(Body::from(logs))) } else { let text = sqlx::query!( - "SELECT created_by AS \"created_by!\", CONCAT(coalesce(v2_as_queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index - FROM v2_as_queue - LEFT JOIN job_logs ON job_logs.job_id = v2_as_queue.id - WHERE v2_as_queue.id = $1 AND v2_as_queue.workspace_id = $2 AND ($3::text[] IS NULL OR v2_as_queue.tag = ANY($3))", + "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index + FROM v2_job j + LEFT JOIN job_logs ON job_logs.job_id = j.id + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", id, w_id, tags.as_ref().map(|v| v.as_slice()) @@ -1613,9 +1585,9 @@ async fn get_args( .map(|authed| get_scope_tags(authed)) .flatten(); let record = sqlx::query!( - "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\" - FROM v2_as_completed_job - WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "SELECT j.created_by AS \"created_by!\", j.args as \"args: sqlx::types::Json>\" + FROM v2_job j + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", id, &w_id, tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, @@ -2170,7 +2142,7 @@ async fn cancel_selection( let mut tx = user_db.begin(&authed).await?; let tags = get_scope_tags(&authed).map(|v| v.iter().map(|s| s.to_string()).collect_vec()); let jobs_to_cancel = sqlx::query_scalar!( - "SELECT id AS \"id!\" FROM v2_as_queue WHERE id = ANY($1) AND schedule_path IS NULL AND ($2::text[] IS NULL OR tag = ANY($2))", + "SELECT j.id AS \"id!\" FROM v2_job j WHERE j.id = ANY($1) AND j.trigger_kind != 'schedule'::job_trigger_kind AND ($2::text[] IS NULL OR j.tag = ANY($2))", &jobs, tags.as_ref().map(|v| v.as_slice()) ) @@ -2274,7 +2246,7 @@ async fn count_queue_jobs( Ok(Json( sqlx::query_as!( QueueStats, - "SELECT coalesce(COUNT(*) FILTER(WHERE suspend = 0 AND running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE suspend > 0), 0) as \"suspended!\" FROM v2_as_queue WHERE (workspace_id = $1 OR $2) AND scheduled_for <= now() AND ($3::text[] IS NULL OR tag = ANY($3))", + "SELECT coalesce(COUNT(*) FILTER(WHERE q.suspend = 0 AND q.running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE q.suspend > 0), 0) as \"suspended!\" FROM v2_job_queue q JOIN v2_job j USING (id) WHERE (j.workspace_id = $1 OR $2) AND q.scheduled_for <= now() AND ($3::text[] IS NULL OR j.tag = ANY($3))", w_id, w_id == "admins" && cq.all_workspaces.unwrap_or(false), tags.as_ref().map(|v| v.as_slice()) @@ -2717,9 +2689,9 @@ async fn get_suspended_flow_info<'c>( let flow = sqlx::query_as!( FlowInfo, r#" - SELECT id AS "id!", flow_status, suspend AS "suspend!", script_path - FROM v2_as_queue - WHERE id = $1 + SELECT j.id AS "id!", COALESCE(s.flow_status, s.workflow_as_code_status) as flow_status, q.suspend AS "suspend!", j.runnable_path as script_path + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE j.id = $1 "#, job_id, ) @@ -2933,9 +2905,9 @@ pub async fn get_flow_user_state( let mut tx = user_db.begin(&authed).await?; let r = sqlx::query_scalar!( r#" - SELECT flow_status->'user_states'->$1 - FROM v2_as_queue - WHERE id = $2 AND workspace_id = $3 + SELECT COALESCE(s.flow_status, s.workflow_as_code_status)->'user_states'->$1 + FROM v2_job_queue q LEFT JOIN v2_job_status s USING (id) + WHERE q.id = $2 AND q.workspace_id = $3 "#, key, job_id, @@ -3864,6 +3836,7 @@ async fn batch_rerun_handle_job( batch_rerun_compute_js_expression(expr.clone(), job.clone()).await?, ); } + InputTransform::Ai => {} } } @@ -4166,10 +4139,10 @@ pub async fn restart_flow( let mut tx = user_db.clone().begin(&authed).await?; let completed_job = sqlx::query!( "SELECT - script_path, args AS \"args: sqlx::types::Json>>\", - tag AS \"tag!\", priority - FROM v2_as_completed_job - WHERE id = $1 and workspace_id = $2", + j.runnable_path as script_path, j.args AS \"args: sqlx::types::Json>>\", + j.tag AS \"tag!\", j.priority + FROM v2_job j + WHERE j.id = $1 and j.workspace_id = $2", job_id, &w_id, ) @@ -4321,7 +4294,12 @@ pub async fn run_script_by_path_inner( tag, timeout, None, - None, + // If the job has a parent job, set priority to 2 as it may be ran synchronously and block a current worker until being executed. Flow steps have a priority of 1 so this is higher. + if run_query.parent_job.is_some() || run_query.root_job.is_some() { + Some(2) + } else { + None + }, push_authed.as_ref(), false, None, @@ -4819,10 +4797,9 @@ pub async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<() pub async fn check_queue_too_long(db: &DB, queue_limit: Option) -> error::Result<()> { if let Some(limit) = queue_limit { let count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM v2_as_queue WHERE canceled = false AND (scheduled_for <= now() - OR (suspend_until IS NOT NULL - AND ( suspend <= 0 - OR suspend_until <= now())))", + "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now() + OR (q.suspend_until IS NOT NULL + AND (q.suspend <= 0 OR q.suspend_until <= now())))", ) .fetch_one(db) .await? @@ -6352,7 +6329,15 @@ async fn run_preview_flow_job( check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); - let (uuid, tx) = push( + let chat_input_enabled = raw_flow.value.chat_input_enabled.unwrap_or(false); + let flow_path = raw_flow.path.clone().unwrap_or_default(); + let user_message = raw_flow + .args + .as_ref() + .and_then(|args| args.get("user_message")) + .cloned(); + + let (uuid, mut tx) = push( &db, tx, &w_id, @@ -6386,6 +6371,25 @@ async fn run_preview_flow_job( None, ) .await?; + + // Set memory_id if provided (for agent memory) + if let Some(memory_id) = run_query.memory_id { + set_flow_memory_id(&mut tx, uuid, memory_id).await?; + } + + // Handle conversation messages for chat-enabled flows + if chat_input_enabled { + handle_chat_conversation_messages( + &mut tx, + &authed, + &w_id, + &flow_path, + &run_query, + user_message.as_ref(), + ) + .await?; + } + tx.commit().await?; Ok((StatusCode::CREATED, uuid.to_string())) @@ -7119,6 +7123,11 @@ pub fn start_job_update_sse_stream( // Update log offset if available if let Some(new_offset) = update.log_offset { if new_offset != log_offset.unwrap_or(0) { + // let logs = update.new_logs.clone().unwrap_or_default(); + // tracing::error!( + // "new_offset: {new_offset}; log_offset: {log_offset:?}; {} {logs}", + // logs.len() + // ); log_offset = Some(new_offset); } else { update.log_offset = None; @@ -7460,7 +7469,10 @@ async fn get_job_update_data( } let job = if record.completed.unwrap_or(false) && get_full_job_on_completion { - let get = GetQuery::new().with_auth(&opt_authed).without_logs(); + let get = GetQuery::new() + .with_auth(&opt_authed) + .without_logs() + .without_code(); Some(get.fetch(&db, job_id, &w_id).await?) } else { None @@ -7963,10 +7975,10 @@ async fn count_by_tag( let counts = sqlx::query_as!( TagCount, r#" - SELECT tag as "tag!", COUNT(*) as "count!" - FROM v2_as_completed_job - WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2) - GROUP BY tag + SELECT j.tag as "tag!", COUNT(*) as "count!" + FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE c.started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR j.workspace_id = $2) + GROUP BY j.tag ORDER BY "count!" DESC "#, horizon as f64, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 802f6f0353..cd2f963680 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -62,7 +62,10 @@ use tower_http::{ }; use windmill_common::db::UserDB; use windmill_common::worker::CLOUD_HOSTED; -use windmill_common::{utils::{configure_client, GIT_VERSION}, BASE_URL, INSTANCE_NAME}; +use windmill_common::{ + utils::{configure_client, GIT_VERSION}, + BASE_URL, INSTANCE_NAME, +}; use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; @@ -180,6 +183,7 @@ pub mod workspaces_ee; mod workspaces_export; mod workspaces_extra; mod workspaces_oss; +mod public_app_layer; #[cfg(feature = "mcp")] mod mcp; @@ -699,6 +703,15 @@ pub async fn run_server( ) }; + let app = if let Some(domain) = public_app_layer::PUBLIC_APP_DOMAIN.as_ref() { + tracing::info!("Public app domain filter enabled for domain: {}", domain); + app.layer(axum::middleware::from_fn( + public_app_layer::public_app_domain_filter, + )) + } else { + app + }; + let app = app.layer(CatchPanicLayer::custom(|err| { tracing::error!("panic in handler, returning 500: {:?}", err); Response::builder() diff --git a/backend/windmill-api/src/public_app_layer.rs b/backend/windmill-api/src/public_app_layer.rs new file mode 100644 index 0000000000..7d5e5a6e3b --- /dev/null +++ b/backend/windmill-api/src/public_app_layer.rs @@ -0,0 +1,229 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::body::Body; +use axum::response::Response; +use http::StatusCode; + +lazy_static::lazy_static! { + pub static ref PUBLIC_APP_DOMAIN: Option = std::env::var("PUBLIC_APP_DOMAIN").ok(); +} + +/// Middleware to restrict public app domain to whitelisted routes +pub async fn public_app_domain_filter( + req: axum::extract::Request, + next: axum::middleware::Next, +) -> Response { + if let Some(public_domain) = PUBLIC_APP_DOMAIN.as_ref() { + if let Some(host) = req.headers().get(http::header::HOST) { + if let Ok(host_str) = host.to_str() { + // Extract hostname without port + let hostname = host_str.split(':').next().unwrap_or(host_str); + + if hostname == public_domain { + let path = req.uri().path(); + + // Check if route is whitelisted + let is_whitelisted = is_public_route_whitelisted(path); + + if !is_whitelisted { + tracing::warn!( + "Rejected request to {} on public app domain {}", + path, + public_domain + ); + return Response::builder() + .status(StatusCode::FORBIDDEN) + .body(Body::from( + "Access forbidden: route not allowed on public app domain", + )) + .unwrap(); + } + } + } + } + } + next.run(req).await +} + +fn is_public_route_whitelisted(path: &str) -> bool { + // Whitelisted route patterns for public app domain + let whitelist = [ + // Public app routes + "/api/ee_license", + "/api/w/*/users/whoami", + "/api/w/*/apps_u/*", + "/api/w/*/resources/list", + "/api/w/*/jobs_u/getupdate_sse/*", + "/api/auth/login", + "/api/w/*/folders/listnames", + "/api/w/*/resources/exists/*", + "/api/w/*/resources/type/get/*", + "/api/w/*/resources/type/listnames", + "/api/oauth/login/*", + "/api/oauth/connect/*", + "/oauth/callback/*", + "/user/login_callback/*", + "/api/workspaces/users", + "/api/users/whoami", + "/api/oauth/list_connects", + "/api/oauth/list_logins", + "/public/*", + "/a/*", + "/Inter-Variable.woff2", + ]; + + for pattern in &whitelist { + if matches_pattern(path, pattern) { + return true; + } + } + + false +} + +fn matches_pattern(path: &str, pattern: &str) -> bool { + // Handle exact match + if path == pattern { + return true; + } + + // Handle wildcard patterns + if !pattern.contains('*') { + return false; + } + + let parts: Vec<&str> = pattern.split('*').collect(); + + // Start from the beginning - path must start with first part + if !path.starts_with(parts[0]) { + return false; + } + + let mut remaining = &path[parts[0].len()..]; + + // Check each part in between wildcards + for i in 1..parts.len() - 1 { + let part = parts[i]; + if let Some(pos) = remaining.find(part) { + remaining = &remaining[pos + part.len()..]; + } else { + return false; + } + } + + // Check the last part + let last = parts[parts.len() - 1]; + if last.is_empty() { + // Wildcard at the end, already matched + return true; + } else { + // Must end with the last part + return remaining.ends_with(last); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_matches_pattern_exact() { + assert!(matches_pattern("/api/version", "/api/version")); + assert!(!matches_pattern("/api/versions", "/api/version")); + } + + #[test] + fn test_matches_pattern_wildcard_middle() { + assert!(matches_pattern( + "/api/w/my-workspace/apps_u/my-app", + "/api/w/*/apps_u/*" + )); + assert!(matches_pattern( + "/api/w/workspace123/jobs_u/job456", + "/api/w/*/jobs_u/*" + )); + assert!(!matches_pattern( + "/api/w/workspace/apps/my-app", + "/api/w/*/apps_u/*" + )); + } + + #[test] + fn test_matches_pattern_wildcard_end() { + assert!(matches_pattern("/api/r/some/path", "/api/r/*")); + assert!(matches_pattern("/api/auth/login", "/api/auth/*")); + assert!(matches_pattern("/public/asset.css", "/public/*")); + assert!(!matches_pattern("/api/admin/users", "/api/r/*")); + } + + #[test] + fn test_is_public_route_whitelisted() { + // Whitelisted routes based on current whitelist + assert!(is_public_route_whitelisted("/api/ee_license")); + assert!(is_public_route_whitelisted( + "/api/w/my-workspace/users/whoami" + )); + assert!(is_public_route_whitelisted( + "/api/w/my-workspace/apps_u/my-app" + )); + assert!(is_public_route_whitelisted( + "/api/w/workspace/resources/list" + )); + + // Non-whitelisted routes + assert!(!is_public_route_whitelisted("/api/version")); + assert!(!is_public_route_whitelisted("/api/uptodate")); + assert!(!is_public_route_whitelisted("/api/w/workspace/scripts")); + assert!(!is_public_route_whitelisted("/api/w/workspace/flows")); + assert!(!is_public_route_whitelisted( + "/api/w/workspace/apps/private" + )); + assert!(!is_public_route_whitelisted("/api/admin/settings")); + assert!(!is_public_route_whitelisted("/api/users")); + } + + #[test] + fn test_matches_pattern_complex_multiple_wildcards() { + // Test pattern with two wildcards like "/api/w/*/foo/*" + assert!(matches_pattern( + "/api/w/workspace1/s3_proxy/file.txt", + "/api/w/*/s3_proxy/*" + )); + assert!(matches_pattern( + "/api/w/my-workspace/s3_proxy/path/to/file.jpg", + "/api/w/*/s3_proxy/*" + )); + assert!(matches_pattern( + "/api/w/test/apps_u/myapp", + "/api/w/*/apps_u/*" + )); + + // Should not match if middle part is different + assert!(!matches_pattern( + "/api/w/workspace1/other/file.txt", + "/api/w/*/s3_proxy/*" + )); + assert!(!matches_pattern( + "/api/w/workspace/apps/myapp", + "/api/w/*/apps_u/*" + )); + + // Edge cases + assert!(matches_pattern( + "/api/w/ws/s3_proxy/a", + "/api/w/*/s3_proxy/*" + )); + + // Path must have content after the last wildcard segment + assert!(matches_pattern( + "/api/w/ws/s3_proxy/", + "/api/w/*/s3_proxy/*" + )); + } +} diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 7e6e6cb6f1..ec1eafe08a 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -798,13 +798,14 @@ async fn delete_resource( check_scopes(&authed, || format!("resources:write:{}", path))?; let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - "DELETE FROM resource WHERE path = $1 AND workspace_id = $2", + let deleted_path = sqlx::query_scalar!( + "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", path, w_id ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .await?; + not_found_if_none(deleted_path, "Resource", &path)?; sqlx::query!( "DELETE FROM variable WHERE path = $1 AND workspace_id = $2", path, @@ -1248,13 +1249,16 @@ async fn delete_resource_type( let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2", + let deleted_name = sqlx::query_scalar!( + "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2 RETURNING name", name, w_id ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .await?; + + not_found_if_none(deleted_name, "ResourceType", &name)?; + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 9a30a70b5f..a639c58f6e 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -131,6 +131,7 @@ pub fn global_service() -> Router { .route("/hub/top", get(get_top_hub_scripts)) .route("/hub/get/*path", get(get_hub_script_by_path)) .route("/hub/get_full/*path", get(get_full_hub_script_by_path)) + .route("/hub/pick/*path", get(pick_hub_script_by_path)) } pub fn global_unauthed_service() -> Router { @@ -606,8 +607,10 @@ async fn create_script_internal<'c>( Ok(None) } (Some(p_hash), o) => { + // Lock the parent row to prevent concurrent updates with the same parent_hash + // This ensures linear lineage - only one script can have a given parent at a time if sqlx::query_scalar!( - "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2", + "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2 FOR UPDATE", p_hash.0, &w_id ) @@ -1112,6 +1115,40 @@ pub async fn get_full_hub_script_by_path( )) } +pub async fn pick_hub_script_by_path( + Path(path): Path, + Extension(db): Extension, +) -> impl IntoResponse { + let path_str = path.to_path(); + + // Extract version_id from path (format: {hub}/{version_id}/{summary}) + let version_id = path_str.split('/').nth(1).unwrap_or(""); + + let hub_base_url = HUB_BASE_URL.read().await.clone(); + + // Determine which hub to use based on version_id + // If version_id < PRIVATE_HUB_MIN_VERSION, use default hub + let target_hub_url = if version_id + .parse::() + .is_ok_and(|v| v < windmill_common::PRIVATE_HUB_MIN_VERSION) + { + windmill_common::DEFAULT_HUB_BASE_URL + } else { + &hub_base_url + }; + + // Call the hub's pick endpoint: /scripts/{version_id}/pick + let (status_code, headers, response) = query_elems_from_hub( + &HTTP_CLIENT, + &format!("{}/scripts/{}/pick", target_hub_url, version_id), + None, + &db, + ) + .await?; + + Ok::<_, Error>((status_code, headers, response)) +} + async fn get_script_by_path( authed: ApiAuthed, Extension(user_db): Extension, diff --git a/backend/windmill-api/src/triggers/trigger_helpers.rs b/backend/windmill-api/src/triggers/trigger_helpers.rs index f813f6ab1b..7deaf6dc1d 100644 --- a/backend/windmill-api/src/triggers/trigger_helpers.rs +++ b/backend/windmill-api/src/triggers/trigger_helpers.rs @@ -46,7 +46,7 @@ struct ScriptInfo { #[derive(Debug, Deserialize)] struct PropertyDefinition { - r#type: Option, + r#type: Option>, } #[derive(Debug, Deserialize)] @@ -119,7 +119,11 @@ fn runnable_format_from_schema_without_preprocessor( if schema.as_ref().is_some_and(|schema| { schema.properties.as_ref().is_some_and(|properties| { properties.iter().any(|(key, def)| { - key == "payload" && def.r#type.as_ref().is_some_and(|t| t == "array") + key == "payload" + && def.r#type.as_ref().is_some_and(|t| { + let typ = t.get().trim(); + typ == "array" || (typ.starts_with('[') && typ.ends_with(']')) + }) }) }) }) => diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 165316ba56..9021d6fe8e 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -494,14 +494,13 @@ async fn list_user_usage( UserWithUsage, " SELECT usr.email, usage.executions - FROM usr - , LATERAL ( - SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions - FROM v2_as_completed_job - WHERE workspace_id = $1 - AND job_kind NOT IN ('flow', 'flowpreview', 'flownode') - AND email = usr.email - AND now() - '1 week'::interval < created_at + FROM usr, LATERAL ( + SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions + FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE j.workspace_id = $1 + AND j.kind NOT IN ('flow', 'flowpreview', 'flownode') + AND j.permissioned_as_email = usr.email + AND now() - '1 week'::interval < j.created_at ) usage WHERE workspace_id = $1 ", diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index d785116d40..c7053209da 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -50,7 +50,7 @@ use windmill_common::{ oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH, utils::{paginate, rd_string, require_admin, Pagination}, }; -use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +use windmill_git_sync::{handle_fork_branch_creation, handle_deployment_metadata, DeployedObject}; use windmill_worker::scoped_dependency_map::{DependencyMap, ScopedDependencyMap}; #[cfg(feature = "enterprise")] @@ -163,6 +163,7 @@ pub fn workspaced_service() -> Router { post(acknowledge_all_critical_alerts), ) .route("/critical_alerts/mute", post(mute_critical_alerts)) + .route("/create_workspace_fork_branch", post(create_workspace_fork_branch)) .route("/operator_settings", post(update_operator_settings)); #[cfg(all(feature = "stripe", feature = "enterprise"))] @@ -3026,19 +3027,33 @@ async fn deprecated_create_workspace_fork(_authed: ApiAuthed) -> Result return Err(Error::BadRequest("This API endpoint has been relocated. Your Windmill CLI version is outdated and needs to be updated.".to_string())); } +/// Return the uuids of the git sync jobs to create the branch before creating the fork +async fn create_workspace_fork_branch( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(nw): Json, +) -> JsonResult> { + if *CLOUD_HOSTED { + return Err(Error::BadRequest(format!( + "Forking workspaces is not available on app.windmill.dev" + ))); + } + + Ok(Json( + handle_fork_branch_creation(&authed.email, &authed.username, &db, &w_id, &nw.id).await?, + )) +} + async fn create_workspace_fork( authed: ApiAuthed, Extension(db): Extension, Path(parent_workspace_id): Path, Json(nw): Json, ) -> Result { - // if *CREATE_WORKSPACE_REQUIRE_SUPERADMIN { - // require_super_admin(&db, &authed.email).await?; - // } - if *CLOUD_HOSTED { return Err(Error::BadRequest(format!( - "Forking workspaces is not available on Cloud" + "Forking workspaces is not available on app.windmill.dev" ))); } diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 3d1524916a..c7972d3a7e 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -433,9 +433,9 @@ pub mod flow { make_static! { /// Flow node cache. /// FIXME: Use `Arc` for cheap cloning. - static ref NODES: { FlowNodeId => RawData } in "flow" <= 1000; + static ref NODES: { FlowNodeId => RawData } in "flow_2" <= 1000; /// Flow version value cache (version id => value). - static ref FLOWS: { i64 => Entry } in "flows" <= 1000; + static ref FLOWS: { i64 => Entry } in "flows_2" <= 1000; /// Flow version lite value cache (version id => value). static ref FLOWS_LITE: { i64 => Entry } in "flowslite" <= 1000; } @@ -567,7 +567,7 @@ pub mod script { make_static! { /// Scripts cache. /// FIXME: Use `Arc` for cheap cloning. - static ref CACHE: { ScriptHash => ScriptFull } in "script" <= 1000; + static ref CACHE: { ScriptHash => ScriptFull } in "script_2" <= 1000; } /// Clear the script cache. @@ -670,7 +670,7 @@ pub mod app { make_static! { /// App scripts cache. - static ref CACHE: { AppScriptId => Entry } in "app" <= 1000; + static ref CACHE: { AppScriptId => Entry } in "app_2" <= 1000; } /// Clear the app scripts cache. diff --git a/backend/windmill-common/src/client.rs b/backend/windmill-common/src/client.rs index b05fdfa2f7..146cc98e68 100644 --- a/backend/windmill-common/src/client.rs +++ b/backend/windmill-common/src/client.rs @@ -49,14 +49,7 @@ impl AuthedClient { "{}/api/w/{}/oidc/token/{}", self.base_internal_url, self.workspace, audience ); - let response = self.get(&url, vec![]).await?; - match response.status().as_u16() { - 200u16 => Ok(response - .json::() - .await - .context("decoding oidc token as json string")?), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), - } + make_basic_get_request(self, &url, None, Some("decoding oidc token as json string")).await } pub async fn get_resource_value(&self, path: &str) -> anyhow::Result { @@ -64,14 +57,7 @@ impl AuthedClient { "{}/api/w/{}/resources/get_value/{}", self.base_internal_url, self.workspace, path ); - let response = self.get(&url, vec![]).await?; - match response.status().as_u16() { - 200u16 => Ok(response - .json::() - .await - .context("decoding resource value as json")?), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), - } + make_basic_get_request(self, &url, None, Some("decoding resource value as json")).await } pub async fn get_variable_value(&self, path: &str) -> anyhow::Result { @@ -79,14 +65,7 @@ impl AuthedClient { "{}/api/w/{}/variables/get_value/{}", self.base_internal_url, self.workspace, path ); - let response = self.get(&url, vec![]).await?; - match response.status().as_u16() { - 200u16 => Ok(response - .json::() - .await - .context("decoding variable value as json")?), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), - } + make_basic_get_request(self, &url, None, Some("decoding variable value as json")).await } pub async fn get_resource_value_interpolated( @@ -121,19 +100,34 @@ impl AuthedClient { "{}/api/w/{}/jobs_u/completed/get_result/{}", self.base_internal_url, self.workspace, path ); - let query = if let Some(json_path) = json_path { - vec![("json_path", json_path)] - } else { - vec![] - }; - let response = self.get(&url, query).await?; - match response.status().as_u16() { - 200u16 => Ok(response - .json::() - .await - .context("decoding completed job result as json")?), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), - } + let query = query_from_json_path(json_path); + make_basic_get_request( + self, + &url, + Some(query), + Some("decoding completed job result as json"), + ) + .await + } + + pub async fn get_flow_env_by_flow_job_id( + &self, + root_job_id: &str, + var_name: &str, + json_path: Option, + ) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/jobs/flow_env_by_flow_job_id/{}/{}", + self.base_internal_url, self.workspace, root_job_id, var_name + ); + let query = query_from_json_path(json_path); + make_basic_get_request( + self, + &url, + Some(query), + Some("decoding flow env variable as json"), + ) + .await } pub async fn get_result_by_id( @@ -146,19 +140,14 @@ impl AuthedClient { "{}/api/w/{}/jobs/result_by_id/{}/{}", self.base_internal_url, self.workspace, flow_job_id, node_id ); - let query = if let Some(json_path) = json_path { - vec![("json_path", json_path)] - } else { - vec![] - }; - let response = self.get(&url, query).await?; - match response.status().as_u16() { - 200u16 => Ok(response - .json::() - .await - .context("decoding result by id as json")?), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), - } + let query = query_from_json_path(json_path); + make_basic_get_request( + self, + &url, + Some(query), + Some("decoding result by id as json"), + ) + .await } pub async fn upload_s3_file( @@ -245,3 +234,33 @@ impl AuthedClient { } } } + +#[inline] +fn query_from_json_path(json_path: Option) -> Vec<(&'static str, String)> { + json_path + .map(|json_path| vec![("json_path", json_path)]) + .unwrap_or_else(|| Vec::new()) +} + +#[inline] +async fn make_basic_get_request( + client: &AuthedClient, + url: &str, + query: Option>, + context: Option<&'static str>, +) -> anyhow::Result { + let response = client + .get(&url, query.unwrap_or_else(|| Vec::new())) + .await?; + + match response.status().as_u16() { + 200u16 => { + let json_body = response + .json::() + .await + .context(context.unwrap_or("error decoding body as json"))?; + Ok(json_body) + } + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } +} diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 5c4d6d5d8d..cf230bfddc 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -193,6 +193,8 @@ pub struct FlowValue { #[serde(skip_serializing_if = "Option::is_none")] pub chat_input_enabled: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub flow_env: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] pub notes: Option>, } @@ -565,6 +567,10 @@ impl FlowModule { .map_err(crate::error::to_anyhow) } + pub fn is_ai_agent(&self) -> bool { + self.get_type().is_ok_and(|x| x == "aiagent") + } + pub fn is_simple(&self) -> bool { //todo: flow modules could also be simple execpt for the fact that the case of having single parallel flow approval step is not handled well (Create SuspendedTimeout) self.get_type() @@ -677,6 +683,7 @@ pub enum InputTransform { #[serde(default = "default_empty_string")] expr: String, }, + Ai, } impl InputTransform { @@ -695,6 +702,7 @@ impl TryFrom for InputTransform { let input_transform = match value.type_.as_str() { "static" => InputTransform::new_static_value(value.value.unwrap_or_else(default_null)), "javascript" => InputTransform::new_javascript_expr(&value.expr.unwrap_or_default()), + "ai" => InputTransform::Ai, other => { return Err(anyhow::anyhow!( "got value: {other} for field `type`, expected value: `static` or `javascript`" @@ -850,6 +858,10 @@ pub struct McpToolValue { pub exclude_tools: Vec, } +fn is_none_or_empty_vec(expr: &Option>) -> bool { + expr.is_none() || expr.as_ref().unwrap().is_empty() +} + #[derive(Serialize, Debug, Clone)] #[serde( tag = "type", @@ -941,7 +953,7 @@ pub enum FlowModuleValue { concurrency_time_window_s: Option, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "is_none_or_empty_vec")] assets: Option>, }, @@ -967,7 +979,7 @@ pub enum FlowModuleValue { concurrency_time_window_s: Option, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "is_none_or_empty_vec")] assets: Option>, }, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 53675c6ef4..b0a3f5e4ea 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -53,6 +53,7 @@ pub mod job_metrics; pub mod job_s3_helpers_ee; #[cfg(feature = "parquet")] pub mod job_s3_helpers_oss; +pub mod lockfiles; #[cfg(feature = "private")] pub mod git_sync_ee; @@ -97,6 +98,7 @@ pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5; pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; +pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; pub const SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs #[macro_export] diff --git a/backend/windmill-common/src/lockfiles.rs b/backend/windmill-common/src/lockfiles.rs new file mode 100644 index 0000000000..481caa70c3 --- /dev/null +++ b/backend/windmill-common/src/lockfiles.rs @@ -0,0 +1,17 @@ +use crate::scripts::ScriptLang; + +pub const LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT: &str = "# from requirements.txt"; + +pub fn is_generated_from_raw_requirements( + lang: &Option, + lock: &Option, +) -> bool { + (lang.is_some_and(|v| v == ScriptLang::Bun) + && lock + .as_ref() + .is_some_and(|v| v.contains("generatedFromPackageJson"))) + || (lang.is_some_and(|v| v == ScriptLang::Python3) + && lock + .as_ref() + .is_some_and(|v| v.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT))) +} diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index b6a4fd3391..11d6758caa 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -17,7 +17,7 @@ use crate::{ assets::AssetWithAltAccessType, error::{to_anyhow, Error}, utils::http_get_from_hub, - DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, + DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, PRIVATE_HUB_MIN_VERSION, }; use crate::worker::HUB_CACHE_DIR; @@ -585,7 +585,7 @@ pub async fn get_hub_script_by_path( && path .split("/") .next() - .is_some_and(|x| x.parse::().is_ok_and(|x| x < 10_000_000)) + .is_some_and(|x| x.parse::().is_ok_and(|x| x < PRIVATE_HUB_MIN_VERSION)) { tracing::info!( "Not found on private hub, fallback to default hub for {}", @@ -670,10 +670,9 @@ async fn get_full_hub_script_by_path_inner( Ok(response) => Ok(response), Err(e) => { if hub_base_url != DEFAULT_HUB_BASE_URL - && path - .split("/") - .next() - .is_some_and(|x| x.parse::().is_ok_and(|x| x < 10_000_000)) + && path.split("/").next().is_some_and(|x| { + x.parse::().is_ok_and(|x| x < PRIVATE_HUB_MIN_VERSION) + }) { // TODO: should only fallback to default hub if status is 404 (hub returns 500 currently) tracing::info!( @@ -733,18 +732,32 @@ pub fn hash_script(ns: &NewScript) -> i64 { dh.finish() as i64 } +pub struct ClonedScript { + pub old_script: NewScript, + pub new_hash: i64, +} pub async fn clone_script<'c>( base_hash: ScriptHash, w_id: &str, deployment_message: Option, tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, -) -> crate::error::Result { - let s = - sqlx::query_as::<_, Script>("SELECT * FROM script WHERE hash = $1 AND workspace_id = $2") - .bind(base_hash.0) - .bind(w_id) - .fetch_one(&mut **tx) - .await?; +) -> crate::error::Result { + let s = sqlx::query_as::<_, Script>( + "SELECT * FROM script WHERE hash = $1 AND workspace_id = $2 AND archived = false FOR UPDATE", + ) + .bind(base_hash.0) + .bind(w_id) + .fetch_optional(&mut **tx) + .await?; + + let s = if let Some(s) = s { + s + } else { + return Err(crate::error::Error::NotFound(format!( + "Non-archived script with hash {} not found", + base_hash.0 + ))); + }; let ns = NewScript { path: s.path.clone(), @@ -820,5 +833,5 @@ pub async fn clone_script<'c>( .execute(&mut **tx) .await?; - Ok(new_hash) + Ok(ClonedScript { old_script: ns, new_hash }) } diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 9dd9edc64d..1edd292cbe 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -401,8 +401,7 @@ pub async fn get_reserved_variables( value, description: "Custom workspace environment variable".to_string(), is_custom: true, -}) -).collect() +})).collect() } async fn get_cached_workspace_envs(conn: &Connection, w_id: &str) -> Vec<(String, String)> { @@ -439,7 +438,11 @@ async fn get_cached_workspace_envs(conn: &Connection, w_id: &str) -> Vec<(String custom_envs } -pub async fn get_variable_or_self(path: String, db: &DB, w_id: &str) -> crate::error::Result { +pub async fn get_variable_or_self( + path: String, + db: &DB, + w_id: &str, +) -> crate::error::Result { if !path.starts_with("$var:") { return Ok(path); } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index eafd8cd894..0b0055ed0d 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -7,7 +7,7 @@ use regex::Regex; use reqwest_middleware::ClientWithMiddleware; use semver::Version; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use serde_json::value::RawValue; +use serde_json::{json, value::RawValue}; use sqlx::{types::Json, Pool, Postgres}; use std::{ cmp::Reverse, @@ -498,7 +498,6 @@ pub async fn store_pull_query(wc: &WorkerConfig) { pub const TMP_DIR: &str = "/tmp/windmill"; pub const TMP_LOGS_DIR: &str = concatcp!(TMP_DIR, "/logs"); -pub const TMP_MEMORY_DIR: &str = concatcp!(TMP_DIR, "/memory"); pub const HUB_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "hub"); @@ -664,13 +663,11 @@ fn parse_file(path: &str) -> Option { .flatten() } -#[derive(Copy, Clone)] #[annotations("#")] pub struct RubyAnnotations { pub verbose: bool, } -#[derive(Copy, Clone)] #[annotations("#")] pub struct PythonAnnotations { pub no_cache: bool, @@ -683,7 +680,6 @@ pub struct PythonAnnotations { pub py313: bool, } -#[derive(Copy, Clone)] #[annotations("//")] pub struct GoAnnotations { pub go1_22_compat: bool, @@ -699,13 +695,174 @@ pub struct TypeScriptAnnotations { #[annotations("--")] pub struct SqlAnnotations { - pub return_last_result: bool, + pub return_last_result: bool, // deprecated, use result_collection instead + pub result_collection: SqlResultCollectionStrategy, } #[annotations("#")] pub struct BashAnnotations { pub docker: bool, } + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SqlResultCollectionStrategy { + LastStatementAllRows, + LastStatementFirstRow, + LastStatementAllRowsScalar, + LastStatementFirstRowScalar, + AllStatementsAllRows, + AllStatementsFirstRow, + AllStatementsAllRowsScalar, + AllStatementsFirstRowScalar, + Legacy, +} + +impl SqlResultCollectionStrategy { + pub fn parse(s: &str) -> Self { + use SqlResultCollectionStrategy::*; + match s { + "last_statement_all_rows" => LastStatementAllRows, + "last_statement_first_row" => LastStatementFirstRow, + "last_statement_all_rows_scalar" => LastStatementAllRowsScalar, + "last_statement_first_row_scalar" => LastStatementFirstRowScalar, + "all_statements_all_rows" => AllStatementsAllRows, + "all_statements_first_row" => AllStatementsFirstRow, + "all_statements_all_rows_scalar" => AllStatementsAllRowsScalar, + "all_statements_first_row_scalar" => AllStatementsFirstRowScalar, + "legacy" => Legacy, + _ => SqlResultCollectionStrategy::default(), + } + } + + pub fn collect_last_statement_only(&self, query_count: usize) -> bool { + use SqlResultCollectionStrategy::*; + match self { + LastStatementAllRows + | LastStatementFirstRow + | LastStatementFirstRowScalar + | LastStatementAllRowsScalar => true, + Legacy => query_count == 1, + _ => false, + } + } + pub fn collect_first_row_only(&self) -> bool { + use SqlResultCollectionStrategy::*; + match self { + LastStatementFirstRow + | LastStatementFirstRowScalar + | AllStatementsFirstRow + | AllStatementsFirstRowScalar => true, + _ => false, + } + } + pub fn collect_scalar(&self) -> bool { + use SqlResultCollectionStrategy::*; + match self { + LastStatementFirstRowScalar + | AllStatementsFirstRowScalar + | LastStatementAllRowsScalar + | AllStatementsAllRowsScalar => true, + _ => false, + } + } + + // This function transforms the shape (e.g Row[][] -> Row) + // It is the responsibility of the executor to avoid fetching unnecessary statements/rows + pub fn collect( + &self, + values: Vec>>, + ) -> error::Result> { + let null = || serde_json::value::RawValue::from_string("null".to_string()).unwrap(); + + let values = if self.collect_last_statement_only(values.len()) { + values.into_iter().rev().take(1).collect() + } else { + values + }; + + let values = if self.collect_first_row_only() { + values + .into_iter() + .map(|rows| rows.into_iter().take(1).collect()) + .collect() + } else { + values + }; + + let values = if self.collect_scalar() { + values + .into_iter() + .map(|rows| { + rows.into_iter() + .map(|row| { + // Take the first value in the object + let record = + match serde_json::from_str(row.get()) { + Ok(serde_json::Value::Object(record)) => record, + Ok(_) => return Err(error::Error::ExecutionErr( + "Could not collect sql scalar value from non-object row" + .to_string(), + )), + Err(e) => { + return Err(error::Error::ExecutionErr(format!( + "Could not collect sql scalar value (failed to parse row): {}", + e + ))) + } + }; + let Some((_, value)) = record.iter().next() else { + return Err(error::Error::ExecutionErr( + "Could not collect sql scalar value from empty row".to_string(), + )); + }; + Ok(serde_json::value::RawValue::from_string( + serde_json::to_string(value).map_err(to_anyhow)?, + ) + .map_err(to_anyhow)?) + }) + .collect::>>() + }) + .collect::>>()? + } else { + values + }; + + match ( + self.collect_last_statement_only(values.len()), + self.collect_first_row_only(), + ) { + (true, true) => { + match values + .into_iter() + .last() + .map(|rows| rows.into_iter().next()) + { + Some(Some(row)) => Ok(row.clone()), + _ => Ok(null()), + } + } + (true, false) => match values.into_iter().last() { + Some(rows) => Ok(to_raw_value(&rows)), + None => Ok(null()), + }, + (false, true) => { + let values = values + .into_iter() + .map(|rows| rows.into_iter().next().unwrap_or_else(null)) + .collect::>(); + Ok(to_raw_value(&values)) + } + (false, false) => Ok(to_raw_value(&values)), + } + } +} + +impl Default for SqlResultCollectionStrategy { + fn default() -> Self { + SqlResultCollectionStrategy::LastStatementAllRows + } +} + /// length = 5 /// value = "foo" /// output = "foo " @@ -1702,6 +1859,13 @@ pub fn load_env_vars( .collect() } +pub fn error_to_value(err: &error::Error) -> serde_json::Value { + match err { + error::Error::JsonErr(err) => err.clone(), + _ => json!({"message": err.to_string(), "name": err.name()}), + } +} + #[derive(Clone, PartialEq, Debug)] pub struct WorkspacedPath { pub workspace_id: String, diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.lock b/backend/windmill-duckdb-ffi-internal/Cargo.lock index b7512f471d..a53a988f30 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.lock +++ b/backend/windmill-duckdb-ffi-internal/Cargo.lock @@ -416,8 +416,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "duckdb" version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a093eed1c714143b257b95fa323e38527fabf05fbf02bb0d5d2045275ffdaef" +source = "git+https://github.com/diegoimbert/duckdb-rs?branch=main#0df52a6941c9996d7ec60d585eaf6430db8c48cf" dependencies = [ "arrow", "cast", @@ -704,8 +703,7 @@ checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libduckdb-sys" version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b93c3ff279601516f01531cadf2ccba50394fbb5f7bf685c6e6b9b07c8dca6f" +source = "git+https://github.com/diegoimbert/duckdb-rs?branch=main#0df52a6941c9996d7ec60d585eaf6430db8c48cf" dependencies = [ "cc", "flate2", diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.toml b/backend/windmill-duckdb-ffi-internal/Cargo.toml index 315055483e..9560e1d0e6 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.toml +++ b/backend/windmill-duckdb-ffi-internal/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] chrono = "0.4.41" -duckdb = { version = "^1.4.1", features = ["bundled"] } +duckdb = { git = "https://github.com/diegoimbert/duckdb-rs", branch = "main", features = ["bundled"] } rust_decimal = "1.37.2" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } diff --git a/backend/windmill-duckdb-ffi-internal/README_DEV.md b/backend/windmill-duckdb-ffi-internal/README_DEV.md index 97dd8a5eb0..0c34ebc069 100644 --- a/backend/windmill-duckdb-ffi-internal/README_DEV.md +++ b/backend/windmill-duckdb-ffi-internal/README_DEV.md @@ -12,4 +12,4 @@ INSERT INTO t VALUES (NULL); causes `Constraint Error: NOT NULL constraint failed: t.x` normally, but here we see `Unknown exception in ExecutorTask::Execute`. This opaque errors comes directly from the C++ DuckDB library : https://github.com/duckdb/duckdb/blob/f99fed1e0b16a842573f9dad529f6c170a004f6e/src/parallel/executor_task.cpp#L58 -To solve this, we compile duckdb separately from the main backend crate and call it with FFI +To solve this, we compile duckdb separately from the main backend crate and call it with FFI. It has to be loaded dynamically, if it is loaded statically it will still share lib c++ with deno_core and cause issues. diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index cb5c43a018..501b5c5dee 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -1,11 +1,11 @@ use std::{ collections::HashMap, - ffi::{c_char, CStr, CString}, + ffi::{CStr, CString, c_char, c_uint}, ptr::null_mut, }; -use duckdb::{params_from_iter, types::TimeUnit, Row}; -use rust_decimal::{prelude::FromPrimitive, Decimal}; +use duckdb::{Row, params_from_iter, types::TimeUnit}; +use rust_decimal::{Decimal, prelude::FromPrimitive}; use serde::Deserialize; use serde_json::value::RawValue; @@ -27,6 +27,14 @@ pub extern "C" fn free_cstr(string: *mut c_char) -> () { } } +#[unsafe(no_mangle)] +pub extern "C" fn get_version() -> c_uint { + // Increment when making breaking changes to the FFI interface. + // The windmill worker will check that the version matches or else refuse to call + // the FFI functions to avoid undefined behavior. + return 1; +} + #[unsafe(no_mangle)] pub extern "C" fn run_duckdb_ffi( query_block_list: *const *const c_char, @@ -36,6 +44,8 @@ pub extern "C" fn run_duckdb_ffi( base_internal_url: *const c_char, w_id: *const c_char, column_order_ptr: *mut *mut c_char, + collect_last_only: bool, + collect_first_row_only: bool, ) -> *mut c_char { let (r, column_order) = match convert_args( query_block_list, @@ -54,6 +64,8 @@ pub extern "C" fn run_duckdb_ffi( token, base_internal_url, w_id, + collect_last_only, + collect_first_row_only, ) }, ) { @@ -153,6 +165,8 @@ fn run_duckdb_internal<'a>( token: &str, base_internal_url: &str, w_id: &str, + collect_last_only: bool, + collect_first_row_only: bool, ) -> Result<(String, Option>), String> { let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?; @@ -189,23 +203,23 @@ fn run_duckdb_internal<'a>( )) .map_err(|e| format!("Error setting up S3 secret: {}", e.to_string()))?; - let mut result: Option> = None; + let mut results: Vec>> = vec![]; let mut column_order = None; for (query_block_index, query_block) in query_block_list.enumerate() { - result = Some( - do_duckdb_inner( - &conn, - query_block, - &job_args, - query_block_index != query_block_list_count - 1, - &mut column_order, - ) - .map_err(|e| e.to_string())?, - ); + let result = do_duckdb_inner( + &conn, + query_block, + &job_args, + collect_last_only && query_block_index != query_block_list_count - 1, + collect_first_row_only, + &mut column_order, + ) + .map_err(|e| e.to_string())?; + results.push(result); } - let result = result.unwrap_or_else(|| RawValue::from_string("[]".to_string()).unwrap()); - Ok((result.get().to_string(), column_order)) + let results = serde_json::value::to_raw_value(&results).map_err(|e| e.to_string())?; + Ok((results.get().to_string(), column_order)) } fn do_duckdb_inner( @@ -213,8 +227,9 @@ fn do_duckdb_inner( query: &str, job_args: &HashMap, skip_collect: bool, + collect_first_row_only: bool, column_order: &mut Option>, -) -> Result, String> { +) -> Result>, String> { let mut rows_vec = vec![]; let (query, job_args) = interpolate_named_args(query, &job_args); @@ -226,17 +241,18 @@ fn do_duckdb_inner( .map_err(|e| e.to_string())?; if skip_collect { - return Ok(RawValue::from_string("[]".to_string()).unwrap()); + return Ok(vec![]); } - // Statement needs to be stepped at least once or stmt.column_names() will panic let mut column_names = None; + let mut type_aliases = None; loop { let row = rows.next(); match row { Ok(Some(row)) => { // Set column names if not already set let stmt = row.as_ref(); + let column_names = match column_names.as_ref() { Some(column_names) => column_names, None => { @@ -244,8 +260,20 @@ fn do_duckdb_inner( column_names.as_ref().unwrap() } }; + let type_aliases = match type_aliases.as_ref() { + Some(type_aliases) => type_aliases, + None => { + type_aliases = Some( + (0..stmt.column_count()) + .map(|i| stmt.column_logical_type(i).get_alias()) + .collect::>(), + ); + type_aliases.as_ref().unwrap() + } + }; - let row = row_to_value(row, &column_names.as_slice()).map_err(|e| e.to_string())?; + let row = row_to_value(row, &column_names.as_slice(), &type_aliases.as_slice()) + .map_err(|e| e.to_string())?; rows_vec.push(row); } Ok(None) => break, @@ -253,11 +281,14 @@ fn do_duckdb_inner( return Err(e.to_string()); } } + if collect_first_row_only { + break; + } } *column_order = column_names; - serde_json::value::to_raw_value(&rows_vec).map_err(|e| e.to_string()) + Ok(rows_vec) } // duckdb-rs does not support named parameters, @@ -282,83 +313,99 @@ fn interpolate_named_args<'a>( (query, values) } -fn row_to_value(row: &Row<'_>, column_names: &[String]) -> Result, String> { +fn row_to_value( + row: &Row<'_>, + column_names: &[String], + type_aliases: &[Option], +) -> Result, String> { let mut obj = serde_json::Map::new(); for (i, key) in column_names.iter().enumerate() { let value: duckdb::types::Value = row.get(i).map_err(|e| e.to_string())?; - let json_value = match value { - duckdb::types::Value::Null => serde_json::Value::Null, - duckdb::types::Value::Boolean(b) => serde_json::Value::Bool(b), - duckdb::types::Value::TinyInt(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::SmallInt(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::Int(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::BigInt(i) => serde_json::Value::Number(i.into()), - duckdb::types::Value::HugeInt(i) => serde_json::Value::String(i.to_string()), - duckdb::types::Value::UTinyInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::USmallInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::UInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()), - duckdb::types::Value::Float(f) => serde_json::Value::Number( - serde_json::Number::from_f64(f as f64) - .ok_or_else(|| "Could not convert to f64".to_string())?, - ), - duckdb::types::Value::Double(f) => serde_json::Value::Number( - serde_json::Number::from_f64(f) - .ok_or_else(|| "Could not convert to f64".to_string())?, - ), - duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), - duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), - duckdb::types::Value::Text(s) => serde_json::Value::String(s), - duckdb::types::Value::Blob(b) => serde_json::Value::Array( - b.into_iter() - .map(|byte| serde_json::Value::Number(byte.into())) - .collect(), - ), - duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()), - duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()), - duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({ - "months": months, - "days": days, - "nanos": nanos - }), - duckdb::types::Value::List(values) => serde_json::Value::Array( - values - .into_iter() - .map(|v| serde_json::Value::String(format!("{:?}", v))) - .collect(), - ), - duckdb::types::Value::Enum(e) => serde_json::Value::String(e), - duckdb::types::Value::Struct(fields) => serde_json::Value::Object( - fields - .iter() - .map(|(k, v)| (k.clone(), serde_json::Value::String(format!("{:?}", v)))) - .collect(), - ), - duckdb::types::Value::Array(values) => serde_json::Value::Array( - values - .into_iter() - .map(|v| serde_json::Value::String(format!("{:?}", v))) - .collect(), - ), - duckdb::types::Value::Map(map) => serde_json::Value::Object( - map.iter() - .map(|(k, v)| { - ( - format!("{:?}", k), - serde_json::Value::String(format!("{:?}", v)), - ) - }) - .collect(), - ), - duckdb::types::Value::Union(value) => { - serde_json::Value::String(format!("{:?}", *value)) - } - }; + let type_alias = &type_aliases[i]; + let json_value = duckdb_value_to_json_value(value, type_alias)?; obj.insert(key.clone(), json_value); } serde_json::value::to_raw_value(&obj).map_err(|e| e.to_string()) } +fn duckdb_value_to_json_value( + value: duckdb::types::Value, + type_alias: &Option, +) -> Result { + let json_value = match value { + duckdb::types::Value::Null => serde_json::Value::Null, + duckdb::types::Value::Boolean(b) => serde_json::Value::Bool(b), + duckdb::types::Value::TinyInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::SmallInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::Int(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::BigInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::HugeInt(i) => serde_json::Value::String(i.to_string()), + duckdb::types::Value::UTinyInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::USmallInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::UInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::Float(f) => serde_json::Value::Number( + serde_json::Number::from_f64(f as f64) + .ok_or_else(|| "Could not convert to f64".to_string())?, + ), + duckdb::types::Value::Double(f) => serde_json::Value::Number( + serde_json::Number::from_f64(f) + .ok_or_else(|| "Could not convert to f64".to_string())?, + ), + duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), + duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), + duckdb::types::Value::Text(s) if type_alias.as_deref().unwrap_or_default() == "JSON" => { + serde_json::from_str(&s) + .map_err(|e| format!("Error parsing JSON text: {}", e.to_string()))? + } + duckdb::types::Value::Text(s) => serde_json::Value::String(s), + duckdb::types::Value::Blob(b) => serde_json::Value::Array( + b.into_iter() + .map(|byte| serde_json::Value::Number(byte.into())) + .collect(), + ), + duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()), + duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()), + duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({ + "months": months, + "days": days, + "nanos": nanos + }), + duckdb::types::Value::List(values) => serde_json::Value::Array( + values + .into_iter() + .map(|v| duckdb_value_to_json_value(v, &None)) + .collect::, _>>()?, + ), + duckdb::types::Value::Enum(e) => serde_json::Value::String(e), + duckdb::types::Value::Struct(fields) => serde_json::Value::Object( + fields + .iter() + .map(|(k, v)| duckdb_value_to_json_value(v.clone(), &None).map(|v| (k.clone(), v))) + .collect::, _>>()?, + ), + duckdb::types::Value::Array(values) => serde_json::Value::Array( + values + .into_iter() + .map(|v| duckdb_value_to_json_value(v, &None)) + .collect::, _>>()?, + ), + duckdb::types::Value::Map(map) => serde_json::Value::Object( + map.iter() + .map(|(k, v)| { + let k = match k { + duckdb::types::Value::Text(s) | duckdb::types::Value::Enum(s) => s.clone(), + _ => format!("{:?}", k), + }; + duckdb_value_to_json_value(v.clone(), &None).map(|v| (k, v)) + }) + .collect::, _>>()?, + ), + duckdb::types::Value::Union(value) => serde_json::Value::String(format!("{:?}", *value)), + }; + Ok(json_value) +} + fn json_value_to_duckdb_value( json_value: &serde_json::Value, arg_type: &str, diff --git a/backend/windmill-git-sync/src/git_sync_oss.rs b/backend/windmill-git-sync/src/git_sync_oss.rs index 70e2b53640..795e238e9d 100644 --- a/backend/windmill-git-sync/src/git_sync_oss.rs +++ b/backend/windmill-git-sync/src/git_sync_oss.rs @@ -21,3 +21,14 @@ pub async fn handle_deployment_metadata<'c>( // Git sync is an enterprise feature and not part of the open-source version return Ok(()); } + +#[cfg(not(feature = "private"))] +pub async fn handle_fork_branch_creation<'c>( + _email: &str, + _created_by: &str, + _db: &DB, + _w_id: &str, + _fork_workspace_id: &str, +) -> Result> { + return Ok(vec![]); +} diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index 380cebb70b..42ceec8cda 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -12,7 +12,7 @@ use windmill_common::{scripts::ScriptHash, DB}; pub mod git_sync_ee; pub mod git_sync_oss; -pub use git_sync_oss::handle_deployment_metadata; +pub use git_sync_oss::{handle_deployment_metadata, handle_fork_branch_creation}; #[derive(Clone, Debug)] pub enum DeployedObject { diff --git a/backend/windmill-macros/src/lib.rs b/backend/windmill-macros/src/lib.rs index 25ff8cbaa2..778ac72b0f 100644 --- a/backend/windmill-macros/src/lib.rs +++ b/backend/windmill-macros/src/lib.rs @@ -1,93 +1,83 @@ use proc_macro::TokenStream; use quote::quote; -use syn::{parse_macro_input, Ident, ItemStruct, Lit}; +use syn::{parse_macro_input, ItemStruct, Lit, Type}; + +fn is_bool_type(ty: &Type) -> bool { + if let Type::Path(type_path) = ty { + if let Some(segment) = type_path.path.segments.last() { + return segment.ident == "bool"; + } + } + false +} #[proc_macro_attribute] pub fn annotations(attr: TokenStream, item: TokenStream) -> TokenStream { let input = parse_macro_input!(item as ItemStruct); let name = input.ident.clone(); - let fields = input - .fields - .iter() - .map(|f| f.ident.clone().unwrap()) - .collect::>(); - // Match on the literal to extract the string value + // Separate fields by type + let mut bool_fields = Vec::new(); + let mut custom_fields = Vec::new(); + + for field in input.fields.iter() { + let field_name = field.ident.clone().unwrap(); + let field_type = &field.ty; + + if is_bool_type(field_type) { + bool_fields.push(field_name); + } else { + custom_fields.push((field_name, field_type)); + } + } + let (custom_field_names, custom_field_types): (Vec<_>, Vec<_>) = + custom_fields.into_iter().unzip(); + + // Parse comment literal let comm_lit = match parse_macro_input!(attr as Lit) { - Lit::Str(lit_str) => lit_str.value(), // This will give "#" without quotes + Lit::Str(lit_str) => lit_str.value(), _ => panic!("Expected a string literal"), }; - // Generate regex - let mut reg = format!("^{}|", &comm_lit); - { - for field in fields.iter() { - reg.push_str(&(field.to_string())); - reg.push_str("\\b"); - } - - reg.push_str(r#"|\w+"#); - } - // Example of generated regex: - // ^# - // |ann1\b|ann2\b|ann3\b|ann4\b - // |\w+ - TokenStream::from(quote! { - #[derive(Default, Debug)] + #[derive(Default, Debug, Copy, Clone)] #input - impl std::ops::BitOrAssign for #name{ - fn bitor_assign(&mut self, rhs: Self) { - // Unfold fields - // Read more: https://docs.rs/quote/latest/quote/macro.quote.html#interpolation - #( self.#fields |= rhs.#fields; )* - } - } - impl #name { /// Autogenerated by windmill-macros - pub fn parse(inner_content: &str) -> Self{ + pub fn parse(code: &str) -> Self { let mut res = Self::default(); - lazy_static::lazy_static! { - static ref RE: regex::Regex = regex::Regex::new(#reg).unwrap(); - } - // Create lines stream - let mut lines = inner_content.lines(); - 'outer: while let Some(line) = lines.next() { - // If comment sign(s) on the right place - let mut comms = false; - // New instance - // We will apply it if in line only annotations - let mut new = Self::default(); - 'inner: for (i, mat) in RE.find_iter(line).enumerate() { + let mut lines = code.lines(); - match mat.as_str(){ - #comm_lit if i == 0 => { - comms = true; - continue 'inner; - }, - - // Will expand into something like: - // "ann1" => new.ann1 = true, - // "ann2" => new.ann2 = true, - // "ann3" => new.ann3 = true, - #( stringify!(#fields) => new.#fields = true, )* - // Non annotations - _ => continue 'outer, - }; + while let Some(line) = lines.next() { + if !line.starts_with(#comm_lit) { + break; } + let line = line[#comm_lit.len()..].trim(); + let (key, value) = line.split_once('=').unwrap_or((line, "")); - if !comms { - // We dont want to continue if line does not start with # - return res; + + match key { + #( + stringify!(#custom_field_names) => { + if value.is_empty() { + continue; + } + res.#custom_field_names = #custom_field_types::parse(value); + } + )* + #( + stringify!(#bool_fields) => { + res.#bool_fields = true; + } + )* + _ => { + // Unknown key=value annotation + continue; + } } - - // Apply changes - res |= new; } - res } } diff --git a/backend/windmill-macros/tests/annotations.rs b/backend/windmill-macros/tests/annotations.rs index fd430dfaeb..d906b72887 100644 --- a/backend/windmill-macros/tests/annotations.rs +++ b/backend/windmill-macros/tests/annotations.rs @@ -24,7 +24,7 @@ mod annotations_tests { } #[annotations("#")] - #[derive(Eq, PartialEq, Copy, Clone)] + #[derive(Eq, PartialEq)] pub struct Annotations { pub ann1: bool, pub ann2: bool, @@ -34,7 +34,7 @@ mod annotations_tests { } #[annotations("//")] - #[derive(Eq, PartialEq, Copy, Clone)] + #[derive(Eq, PartialEq)] pub struct SlashedAnnotations { pub ann1: bool, pub ann2: bool, @@ -43,7 +43,7 @@ mod annotations_tests { } #[annotations("--")] - #[derive(Eq, PartialEq, Copy, Clone)] + #[derive(Eq, PartialEq)] pub struct MinusedAnnotations { pub ann1: bool, pub ann2: bool, diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index b50773f9c8..6349aee84e 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -45,3 +45,5 @@ axum.workspace = true serde_urlencoded.workspace = true regex.workspace = true backon.workspace = true +quick_cache.workspace = true +thiserror.workspace = true diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 1e68402e30..9b941ce911 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -6,6 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ +use std::future::Future; use std::{collections::HashMap, sync::Arc, vec}; use anyhow::Context; @@ -13,6 +14,7 @@ use async_recursion::async_recursion; use chrono::{DateTime, Utc}; use futures::future::TryFutureExt; use itertools::Itertools; +use quick_cache::sync::Cache; #[cfg(feature = "prometheus")] use prometheus::IntCounter; use regex::Regex; @@ -33,7 +35,7 @@ use windmill_common::add_time; use windmill_common::auth::JobPerms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; -use windmill_common::flow_conversations::{add_message_to_conversation_tx, MessageType}; +use windmill_common::lockfiles::is_generated_from_raw_requirements; use windmill_common::jobs::{JobTriggerKind, EMAIL_ERROR_HANDLER_USER_EMAIL}; use windmill_common::utils::{configure_client, now_from_db}; use windmill_common::worker::{Connection, MIN_VERSION_SUPPORTS_DEBOUNCING, SCRIPT_TOKEN_EXPIRY}; @@ -137,7 +139,7 @@ pub struct CanceledBy { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct JobCompleted { - pub job: Arc, + pub job: MiniCompletedJob, pub preprocessed_args: Option>>, pub result: Arc>, pub result_columns: Option>, @@ -154,18 +156,19 @@ pub struct JobCompleted { pub async fn cancel_single_job<'c>( username: &str, reason: Option, - job_running: Arc, + job_running: QueuedJobV2, w_id: &str, mut tx: Transaction<'c, Postgres>, db: &Pool, force_cancel: bool, ) -> error::Result<(Transaction<'c, Postgres>, Option)> { + + let id = job_running.id; if force_cancel || (job_running.parent_job.is_none() && !job_running.running) { let username = username.to_string(); let w_id = w_id.to_string(); let db = db.clone(); tracing::info!("cancelling job {:?}", job_running.id); - let job_running = job_running.clone(); tokio::task::spawn(async move { let reason: String = reason .clone() @@ -179,10 +182,11 @@ pub async fn cancel_single_job<'c>( &Connection::from(db.clone()), ) .await; + let memory_peak = job_running.memory_peak.unwrap_or(0); let add_job = add_completed_job_error( &db, - &MiniPulledJob::from(&job_running), - job_running.mem_peak.unwrap_or(0), + &MiniCompletedJob::from(job_running), + memory_peak, Some(CanceledBy { username: Some(username.to_string()), reason: Some(reason) }), e, "server", @@ -210,7 +214,7 @@ pub async fn cancel_single_job<'c>( } } - Ok((tx, Some(job_running.id))) + Ok((tx, Some(id))) } pub async fn cancel_job<'c>( @@ -223,26 +227,36 @@ pub async fn cancel_job<'c>( force_cancel: bool, require_anonymous: bool, ) -> error::Result<(Transaction<'c, Postgres>, Option)> { - let job = get_queued_job_tx(id, &w_id, &mut tx).await?; + //TODO fetch mini completed job instead of QueuedJob + let job = get_queued_job_v2(&mut *tx, &id).await?; + if job.is_none() { return Ok((tx, None)); } - if require_anonymous && job.as_ref().unwrap().created_by != "anonymous" { + let mut job = job.unwrap(); + + if require_anonymous && job.created_by != "anonymous" { return Err(Error::BadRequest( "You are not logged in and this job was not created by an anonymous user like you so you cannot cancel it".to_string(), )); } - let mut job = job.unwrap(); + + if job.workspace_id != w_id { + return Err(Error::BadRequest( + "You are not authorized to cancel this job belonging to another workspace".to_string(), + )); + } + if force_cancel { // if force canceling a flow step, make sure we force cancel from the highest parent loop { if job.parent_job.is_none() { break; } - match get_queued_job_tx(job.parent_job.unwrap(), &w_id, &mut tx).await? { + match get_queued_job_v2(&mut *tx, &job.parent_job.unwrap()).await? { Some(j) => { job = j; } @@ -252,7 +266,7 @@ pub async fn cancel_job<'c>( } // prevent cancelling a future tick of a schedule - if let Some(schedule_path) = job.schedule_path.as_ref() { + if let Some(schedule_path) = job.schedule_path().as_ref() { let now = now_from_db(&mut *tx).await?; if job.scheduled_for > now { return Err(Error::BadRequest( @@ -265,7 +279,7 @@ pub async fn cancel_job<'c>( } } - let job = Arc::new(job); + let job = job; // get all children using recursive CTE let mut jobs_to_cancel = sqlx::query!( @@ -307,7 +321,7 @@ ORDER BY depth, id let (ntx, _) = cancel_single_job( username, reason.clone(), - job.clone(), + job, w_id, tx, db, @@ -334,13 +348,13 @@ ORDER BY depth, id } } for job_id in jobs_to_cancel { - let job = get_queued_job_tx(job_id, &w_id, &mut tx).await?; + let job = get_queued_job_v2(&mut *tx, &job_id).await?; if let Some(job) = job { let (ntx, _) = cancel_single_job( username, reason.clone(), - Arc::new(job), + job, w_id, tx, db, @@ -553,7 +567,7 @@ async fn cancel_persistent_script_jobs_internal<'c>( // we could have retrieved the job IDs in the first query where we retrieve the hashes, but just in case a job was inserted in the queue right in-between the two above query, we re-do the fetch here let jobs_to_cancel = sqlx::query_scalar::<_, Uuid>( - "SELECT id FROM v2_as_queue WHERE workspace_id = $1 AND script_path = $2 AND canceled = false", + "SELECT j.id FROM v2_job_queue q JOIN v2_job j USING (id) WHERE j.workspace_id = $1 AND j.runnable_path = $2 AND q.canceled_by IS NULL", ) .bind(w_id) .bind(script_path) @@ -702,7 +716,7 @@ where pub async fn add_completed_job_error( db: &Pool, - queued_job: &MiniPulledJob, + completed_job: &MiniCompletedJob, mem_peak: i32, canceled_by: Option, e: serde_json::Value, @@ -713,7 +727,7 @@ pub async fn add_completed_job_error( #[cfg(feature = "prometheus")] register_metric( &WORKER_EXECUTION_FAILED, - &queued_job.tag, + &completed_job.tag, |s| { let counter = prometheus::register_int_counter!(prometheus::Opts::new( "worker_execution_failed", @@ -732,13 +746,13 @@ pub async fn add_completed_job_error( let result = WrappedError { error: e }; tracing::error!( "job {} in {} did not succeed: {}", - queued_job.id, - queued_job.workspace_id, + completed_job.id, + completed_job.workspace_id, serde_json::to_string(&result).unwrap_or_else(|_| "".to_string()) ); let _ = add_completed_job( db, - &queued_job, + &completed_job, false, false, Json(&result), @@ -757,11 +771,18 @@ pub async fn add_completed_job_error( lazy_static::lazy_static! { pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok(); pub static ref MAX_RESULT_SIZE_MB: usize = std::env::var("MAX_RESULT_SIZE_MB").unwrap_or("500".to_string()).parse().unwrap_or(500); + + // Cache for restart_unless_cancelled flag - keyed by (hash, workspace_id) + static ref RESTART_UNLESS_CANCELLED_CACHE: Cache<(i64, String), bool> = Cache::new(10000); + + // Cache for workspace error handler settings with 60s TTL + // Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, expiry_timestamp) + static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache, Option>>, bool, i64)> = Cache::new(1000); } pub async fn add_completed_job( db: &Pool, - queued_job: &MiniPulledJob, + completed_job: &MiniCompletedJob, success: bool, skipped: bool, result: Json<&T>, @@ -787,7 +808,7 @@ pub async fn add_completed_job( let (opt_uuid, duration, _skip_downstream_error_handlers) = (|| { commit_completed_job( db, - queued_job, + completed_job, success, skipped, result, @@ -823,12 +844,12 @@ pub async fn add_completed_job( } #[cfg(feature = "cloud")] - apply_completed_job_cloud_usage(db, queued_job, duration); + apply_completed_job_cloud_usage(db, completed_job, duration); #[cfg(all(feature = "enterprise", feature = "private"))] crate::jobs_ee::apply_completed_job_error_handlers( db, - queued_job, + completed_job, success, result, &canceled_by, @@ -836,92 +857,17 @@ pub async fn add_completed_job( ) .await; - restart_job_if_perpetual(db, queued_job, &canceled_by).await?; + restart_job_if_perpetual(db, completed_job, &canceled_by).await?; - // Create assistant message if it's a flow and it's done, but only if last module is not an AI agent - if !skipped && flow_is_done { - let chat_input_enabled = queued_job.parse_chat_input_enabled(); - if chat_input_enabled.unwrap_or(false) { - // Get conversation_id from flow_status.memory_id - let flow_status = queued_job.parse_flow_status(); - let conversation_id = flow_status.and_then(|fs| fs.memory_id); - - if let Some(conversation_id) = conversation_id { - // get flow value - let parent_job = queued_job.parent_job.unwrap_or(queued_job.id); - let flow_value = sqlx::query!( - "SELECT f.value as \"value: Json\" FROM v2_job j JOIN flow_version f ON f.id = j.runnable_id WHERE j.id = $1 AND j.workspace_id = $2", - parent_job, - &queued_job.workspace_id - ) - .fetch_optional(db) - .await? - .map(|row| row.value.0); - - let last_module_is_ai_agent = flow_value - .as_ref() - .and_then(|flow_value| { - flow_value.modules.last().and_then(|m| m.get_value().ok()) - }) - .map(|v| matches!(v, FlowModuleValue::AIAgent { .. })) - .unwrap_or(false); - - // Only create assistant message if last module is NOT an AI agent, or there was an error - if !last_module_is_ai_agent || success == false { - let value = serde_json::to_value(result.0).map_err(|e| { - Error::internal_err(format!("Failed to serialize result: {e}")) - })?; - - let content = match value { - // If it's an Object with "output" key AND the output is a String, return it - serde_json::Value::Object(mut map) - if map.contains_key("output") - && matches!( - map.get("output"), - Some(serde_json::Value::String(_)) - ) => - { - if let Some(serde_json::Value::String(s)) = map.remove("output") { - s - } else { - // prettify the whole result - serde_json::to_string_pretty(&map) - .unwrap_or_else(|e| format!("Failed to serialize result: {e}")) - } - } - // Otherwise, if the whole value is a String, return it - serde_json::Value::String(s) => s, - // Otherwise, prettify the whole result - v => serde_json::to_string_pretty(&v) - .unwrap_or_else(|e| format!("Failed to serialize result: {e}")), - }; - - // Insert new assistant message - let mut tx = db.begin().await?; - add_message_to_conversation_tx( - &mut tx, - conversation_id, - Some(queued_job.id), - &content, - MessageType::Assistant, - None, - success, - ) - .await?; - tx.commit().await?; - } - } - } - } // tracing::error!("4 {:?}", start.elapsed()); - Ok((queued_job.id, duration)) + Ok((completed_job.id, duration)) } async fn commit_completed_job( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, success: bool, skipped: bool, result: Json<&T>, @@ -1250,7 +1196,7 @@ async fn commit_completed_job( tracing::info!( %job_id, root_job = ?queued_job.flow_innermost_root_job.map(|x| x.to_string()).unwrap_or_else(|| String::new()), - path = &queued_job.runnable_path(), + path = &queued_job.runnable_path, job_kind = ?queued_job.kind, started_at = ?queued_job.started_at.map(|x| x.to_string()).unwrap_or_else(|| String::new()), duration = ?duration, @@ -1271,7 +1217,7 @@ async fn commit_completed_job( async fn check_result_size( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, result: Json<&T>, ) -> Option, i64, bool), Error>> { let result_size = result.size() / 1024 / 1024; @@ -1307,7 +1253,7 @@ async fn check_result_size( async fn restart_job_if_perpetual( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, canceled_by: &Option, ) -> Result<(), Error> { if !queued_job.is_flow_step() && queued_job.kind == JobKind::Script && canceled_by.is_none() { @@ -1333,18 +1279,27 @@ async fn restart_job_if_perpetual( async fn restart_job_if_perpetual_inner( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, hash: ScriptHash, ) -> Result<(), Error> { - let restart = sqlx::query_scalar!( - "SELECT restart_unless_cancelled FROM script WHERE hash = $1 AND workspace_id = $2", - hash.0, - &queued_job.workspace_id - ) - .fetch_optional(db) - .await? - .flatten() - .unwrap_or(false); + let cache_key = (hash.0, queued_job.workspace_id.clone()); + + let restart = if let Some(cached) = RESTART_UNLESS_CANCELLED_CACHE.get(&cache_key) { + cached + } else { + let restart = sqlx::query_scalar!( + "SELECT restart_unless_cancelled FROM script WHERE hash = $1 AND workspace_id = $2", + hash.0, + &queued_job.workspace_id + ) + .fetch_optional(db) + .await? + .flatten() + .unwrap_or(false); + + RESTART_UNLESS_CANCELLED_CACHE.insert(cache_key, restart); + restart + }; if restart { let tx = PushIsolationLevel::IsolatedRoot(db.clone()); @@ -1364,17 +1319,25 @@ async fn restart_job_if_perpetual_inner( None }; - let ehm = HashMap::new(); + let args = sqlx::query_scalar!( + "SELECT args as \"args: sqlx::types::Json>>\" FROM v2_job WHERE id = $1 AND workspace_id = $2", + queued_job.id, + queued_job.workspace_id + ) + .fetch_optional(db) + .await? + .flatten() + .unwrap_or_default(); let (_uuid, tx) = push( db, tx, &queued_job.workspace_id, JobPayload::ScriptHash { hash, - path: queued_job.runnable_path().to_string(), + path: queued_job.runnable_path.clone().unwrap_or_default(), custom_concurrency_key: custom_concurrency_key(db, &queued_job.id).await?, - concurrent_limit: queued_job.concurrent_limit, - concurrency_time_window_s: queued_job.concurrency_time_window_s, + concurrent_limit: None, + concurrency_time_window_s: None, cache_ttl: queued_job.cache_ttl, dedicated_worker: None, language: queued_job @@ -1387,11 +1350,7 @@ async fn restart_job_if_perpetual_inner( custom_debounce_key: None, debounce_delay_s: None, }, - queued_job - .args - .as_ref() - .map(|x| PushArgs::from(&x.0)) - .unwrap_or_else(|| PushArgs::from(&ehm)), + PushArgs::from(&args.0), &queued_job.created_by, &queued_job.permissioned_as_email, queued_job.permissioned_as.clone(), @@ -1405,9 +1364,9 @@ async fn restart_job_if_perpetual_inner( false, false, None, - queued_job.visible_to_owner, + true, Some(queued_job.tag.clone()), - queued_job.timeout, + None, None, queued_job.priority, None, @@ -1424,7 +1383,7 @@ async fn restart_job_if_perpetual_inner( #[cfg(feature = "cloud")] fn apply_completed_job_cloud_usage( db: &Pool, - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, _duration: i64, ) { if *CLOUD_HOSTED && !queued_job.is_flow() && _duration > 1000 { @@ -1489,7 +1448,7 @@ fn apply_completed_job_cloud_usage( } pub async fn send_error_to_global_handler<'a, T: Serialize + Send + Sync>( - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, db: &Pool, result: Json<&T>, ) -> Result<(), Error> { @@ -1525,7 +1484,7 @@ pub async fn send_error_to_global_handler<'a, T: Serialize + Send + Sync>( } pub async fn report_error_to_workspace_handler_or_critical_side_channel( - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, db: &Pool, error_message: String, ) -> () { @@ -1587,32 +1546,76 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel( } pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>( - queued_job: &MiniPulledJob, + queued_job: &MiniCompletedJob, is_canceled: bool, db: &Pool, result: Json<&'a T>, ) -> Result<(), Error> { let w_id = &queued_job.workspace_id; - let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( - r#" - SELECT - error_handler, - error_handler_extra_args, - error_handler_muted_on_cancel - FROM - workspace_settings - WHERE - workspace_id = $1 - "#, - ) - .bind(&w_id) - .fetch_optional(db) - .await - .context("fetching error handler info from workspace_settings")? - .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; + // Try to get from cache first, checking if entry is still valid (within 60s TTL) + let now = chrono::Utc::now().timestamp(); + let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel) = + if let Some(cached) = WORKSPACE_ERROR_HANDLER_CACHE.get(w_id) { + if cached.3 > now { + // Cache hit and not expired + (cached.0.clone(), cached.1.clone(), cached.2) + } else { + // Cache expired, fetch from database + let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( + r#" + SELECT + error_handler, + error_handler_extra_args, + error_handler_muted_on_cancel + FROM + workspace_settings + WHERE + workspace_id = $1 + "#, + ) + .bind(&w_id) + .fetch_optional(db) + .await + .context("fetching error handler info from workspace_settings")? + .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; - let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel) = row_result; + // Update cache with 60s TTL + let expiry = now + 60; + WORKSPACE_ERROR_HANDLER_CACHE.insert( + w_id.clone(), + (row_result.0.clone(), row_result.1.clone(), row_result.2, expiry) + ); + row_result + } + } else { + // Cache miss, fetch from database + let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( + r#" + SELECT + error_handler, + error_handler_extra_args, + error_handler_muted_on_cancel + FROM + workspace_settings + WHERE + workspace_id = $1 + "#, + ) + .bind(&w_id) + .fetch_optional(db) + .await + .context("fetching error handler info from workspace_settings")? + .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; + + // Store in cache with 60s TTL + let expiry = now + 60; + WORKSPACE_ERROR_HANDLER_CACHE.insert( + w_id.clone(), + (row_result.0.clone(), row_result.1.clone(), row_result.2, expiry) + ); + row_result + }; if is_canceled && error_handler_muted_on_cancel { return Ok(()); @@ -1670,7 +1673,7 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> pub async fn handle_maybe_scheduled_job<'c>( db: &Pool, - job: &MiniPulledJob, + job: &MiniCompletedJob, schedule: &Schedule, script_path: &str, w_id: &str, @@ -1978,6 +1981,142 @@ pub struct MiniPulledJob { pub permissioned_as_end_user_email: Option, } + +#[derive(Debug, Clone, Serialize, Deserialize)] + +pub struct MiniCompletedJob { + pub id: Uuid, + pub workspace_id: String, + pub runnable_id: Option, + pub scheduled_for: chrono::DateTime, + pub parent_job: Option, + // pub root_job: Option, + pub flow_innermost_root_job: Option, + pub runnable_path: Option, + pub kind: JobKind, + pub started_at: Option>, + pub permissioned_as: String, + pub created_by: String, + pub script_lang: Option, + pub permissioned_as_email: String, + pub flow_step_id: Option, + pub trigger_kind: Option, + pub trigger: Option, + pub priority: Option, + pub concurrent_limit: Option, + pub tag: String, + pub cache_ttl: Option, +} + +impl From for MiniCompletedJob { + fn from(job: QueuedJobV2) -> Self { + MiniCompletedJob { + id: job.id, + workspace_id: job.workspace_id, + runnable_id: job.runnable_id, + scheduled_for: job.scheduled_for, + parent_job: job.parent_job, + flow_innermost_root_job: job.flow_innermost_root_job, + runnable_path: job.runnable_path, + kind: job.kind, + started_at: job.started_at, + permissioned_as: job.permissioned_as, + created_by: job.created_by, + script_lang: job.script_lang, + permissioned_as_email: job.permissioned_as_email, + flow_step_id: job.flow_step_id, + trigger_kind: job.trigger_kind, + trigger: job.trigger, + priority: job.priority, + concurrent_limit: job.concurrent_limit, + tag: job.tag, + cache_ttl: job.cache_ttl, + + } + } +} + +impl From for MiniCompletedJob { + fn from(job: MiniPulledJob) -> Self { + MiniCompletedJob { + id: job.id, + workspace_id: job.workspace_id, + runnable_id: job.runnable_id, + scheduled_for: job.scheduled_for, + parent_job: job.parent_job, + // root_job: job.root_job,, + flow_innermost_root_job: job.flow_innermost_root_job, + runnable_path: job.runnable_path, + kind: job.kind, + started_at: job.started_at, + permissioned_as: job.permissioned_as, + created_by: job.created_by, + script_lang: job.script_lang, + permissioned_as_email: job.permissioned_as_email, + flow_step_id: job.flow_step_id, + trigger_kind: job.trigger_kind, + trigger: job.trigger, + priority: job.priority, + concurrent_limit: job.concurrent_limit, + tag: job.tag, + cache_ttl: job.cache_ttl, + } + } +} + +impl From> for MiniCompletedJob { + fn from(job: Arc) -> Self { + MiniCompletedJob { + id: job.id, + workspace_id: job.workspace_id.clone(), + runnable_id: job.runnable_id, + scheduled_for: job.scheduled_for, + parent_job: job.parent_job, + flow_innermost_root_job: job.flow_innermost_root_job, + runnable_path: job.runnable_path.clone(), + kind: job.kind, + started_at: job.started_at, + permissioned_as: job.permissioned_as.clone(), + created_by: job.created_by.clone(), + script_lang: job.script_lang, + permissioned_as_email: job.permissioned_as_email.clone(), + flow_step_id: job.flow_step_id.clone(), + trigger_kind: job.trigger_kind.clone(), + trigger: job.trigger.clone(), + priority: job.priority, + concurrent_limit: job.concurrent_limit, + tag: job.tag.clone(), + cache_ttl: job.cache_ttl, + } + } +} + +impl MiniCompletedJob { + pub fn is_flow_step(&self) -> bool { + self.flow_step_id.is_some() + } + pub fn schedule_path(&self) -> Option { + schedule_path(&self.trigger_kind, &self.trigger) + } + + pub fn is_flow(&self) -> bool { + self.kind.is_flow() + } + + pub fn is_dependency(&self) -> bool { + self.kind.is_dependency() + } + +} + +fn schedule_path(trigger_kind: &Option, trigger: &Option) -> Option { + if trigger_kind.as_ref().is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) { + trigger.clone() + } else { + None + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] struct FlowStatusChatInputEnabled { chat_input_enabled: Option, @@ -2064,15 +2203,7 @@ impl MiniPulledJob { } pub fn schedule_path(&self) -> Option { - if self - .trigger_kind - .as_ref() - .is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) - { - self.trigger.clone() - } else { - None - } + schedule_path(&self.trigger_kind, &self.trigger) } pub async fn mark_as_started_if_step(&self, db: &DB) -> Result<(), Error> { @@ -2269,24 +2400,81 @@ pub async fn get_mini_pulled_job<'c>( Ok(job) } + +pub struct QueuedJobV2 { + pub id: Uuid, + pub workspace_id: String, + pub runnable_id: Option, + pub scheduled_for: chrono::DateTime, + pub parent_job: Option, + // pub root_job: Option, + pub flow_innermost_root_job: Option, + pub runnable_path: Option, + pub kind: JobKind, + pub started_at: Option>, + pub permissioned_as: String, + pub created_by: String, + pub script_lang: Option, + pub permissioned_as_email: String, + pub flow_step_id: Option, + pub trigger_kind: Option, + pub trigger: Option, + pub priority: Option, + pub concurrent_limit: Option, + pub tag: String, + pub cache_ttl: Option, + pub last_ping: Option>, + pub worker: Option, + pub memory_peak: Option, + pub running: bool, +} + +impl QueuedJobV2 { + pub fn schedule_path(&self) -> Option { + schedule_path(&self.trigger_kind, &self.trigger) + } +} + +pub async fn get_queued_job_v2<'c>( + e: impl PgExecutor<'c>, job_id: &Uuid) -> error::Result> { + let job = sqlx::query_as!( + QueuedJobV2, + "SELECT id, q.workspace_id, j.runnable_id as \"runnable_id: ScriptHash\", scheduled_for, parent_job, flow_innermost_root_job, runnable_path, kind as \"kind: JobKind\", started_at, permissioned_as, created_by, script_lang as \"script_lang: ScriptLang\", + permissioned_as_email, flow_step_id, trigger_kind as \"trigger_kind: JobTriggerKind\", trigger, q.priority, concurrent_limit, q.tag, cache_ttl, r.ping as last_ping, worker, memory_peak, running + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE j.id = $1", + job_id, + + ) + .fetch_optional(e) + .await?; + Ok(job) +} + #[derive(Serialize, Deserialize, Debug)] pub struct PulledJobResult { pub job: Option, pub suspended: bool, pub missing_concurrency_key: bool, + pub error_while_preprocessing: Option } +#[derive(thiserror::Error, Debug)] pub enum PulledJobResultToJobErr { + #[error("missing concurrency key")] MissingConcurrencyKey(JobCompleted), + #[error("pulled job preprocessor error: {}", .0.result)] + ErrorWhilePreprocessing(JobCompleted), } + impl PulledJobResult { pub fn to_pulled_job(self) -> Result, PulledJobResultToJobErr> { match self { PulledJobResult { job: Some(job), missing_concurrency_key: true, .. } => Err( PulledJobResultToJobErr::MissingConcurrencyKey(JobCompleted { preprocessed_args: None, - job: Arc::new(job.job), + job: MiniCompletedJob::from(job.job), success: false, result: Arc::new(windmill_common::worker::to_raw_value(&json!({ "name": "InternalErr", @@ -2302,9 +2490,278 @@ impl PulledJobResult { from_cache: None, }), ), + PulledJobResult { job: Some(job), error_while_preprocessing: Some(e), .. } => Err( + PulledJobResultToJobErr::ErrorWhilePreprocessing(JobCompleted { + preprocessed_args: None, + job: MiniCompletedJob::from(job.job), + success: false, + result: Arc::new(windmill_common::worker::to_raw_value(&json!({ + "name": "Pulled job preprocessing error", + "message": e + }))), + result_columns: None, + mem_peak: 0, + cached_res_path: None, + token: "".to_string(), + canceled_by: None, + duration: None, + has_stream: Some(false), + from_cache: None, + }), + ), PulledJobResult { job, .. } => Ok(job), } } + + /// Generic preprocess function + /// Can be used for any kind of preprocessing + pub async fn preprocess(&mut self, db: &DB) -> error::Result<()> { + let PulledJobResult { job: Some(ref mut pulled_job), .. } = self else { + return Ok(()); + }; + + let kind = pulled_job.kind; + // Handle dependency job debouncing cleanup when a job is pulled for execution + if kind.is_dependency() + && pulled_job + .args + .as_ref() + .map(|x| x.get("triggered_by_relative_import").is_some()) + .unwrap_or_default() + && !*WMDEBUG_NO_DJOB_DEBOUNCING + { + return Box::pin(async move { + // Only used for testing in tests/relative_imports.rs + // Give us some space to work with. + #[cfg(debug_assertions)] + if let Some(duration) = pulled_job + .args + .as_ref() + .map(|x| { + x.get("dbg_sleep_between_pull_and_debounce_key_removal") + .map(|v| serde_json::from_str::(v.get()).ok()) + .flatten() + }) + .flatten() + { + tracing::debug!("going to sleep",); + sleep(std::time::Duration::from_secs(duration as u64)).await; + } + + tracing::debug!( + "Processing debounce cleanup for dependency job {} at path {:?}", + &pulled_job.id, + &pulled_job.runnable_path + ); + + let key = format!("{}:{}:dependency", &pulled_job.workspace_id, pulled_job.runnable_path()); + let mut tx = db.begin().await?; + + // === DEBOUNCE CLEANUP === + // + // Clean up the debounce_key entry for this job (if it exists). + // + // IMPORTANT: We delete by key (not job_id) to avoid race conditions: + // If pusher has locked this row then this call will be blocked until all txs are commited. + // + // The idea is that the worker_lockfiles::trigger_dependents_to_recompute_locks will fetch the latest version of the obj. + // This object needs to be created before the djob is executed and it happens right here. + // + // This way the next pusher can fetch the latest version of object and base their djob payload on newest version. + // The concurrency limit on djobs will make sure that by the time next djob is started executing the base version it is referencing + // has already calculated all locks. This way even next djob will always use the fully finalized version of object. + // + // + // + // Note: We don't use a transaction here for performance (it's called during job pull). + // This means there's a tiny window where the job is running but key isn't deleted yet, + // which is acceptable because new requests will just accumulate data to this job. + tracing::debug!( + job_id = %pulled_job.id, + "Cleaning up debounce_key entry for completed/pulled job" + ); + + // This will either: + // 1. Block until pusher pushed. Which gives us: + // - If there was any stale data in pusher, then we will read it here (couple of lines below) + // 2. Block pusher until we are done here. This gives us: + // - We will clone objects and retrieve the latest version. So when we are done the pusher can read latest version. + sqlx::query!("DELETE FROM debounce_key WHERE key = $1", &key) + .execute(&mut *tx) + .await + .map_err(|e| { + tracing::error!( + error = %e, + job_id = %pulled_job.id, + "Failed to delete debounce_key" + ); + e + })?; + + let Some(base_hash) = pulled_job.runnable_id else { + return Err(Error::InternalErr( + "Missing runnable_id for dependency job triggered by relative import" + .to_string(), + )); + }; + + tracing::debug!( + job_id = %pulled_job.id, + base_hash = %base_hash, + job_kind = ?kind, + "Creating new version for dependency job triggered by relative import" + ); + + let new_id = match kind { + JobKind::Dependencies => { + let deployment_message = pulled_job + .args + .clone() + .map(|hashmap| { + hashmap + .get("deployment_message") + .map(|map_value| { + serde_json::from_str::(map_value.get()).ok() + }) + .flatten() + }) + .flatten(); + + // This way we tell downstream which script we should archive when the resolution is finished. + // (not used at the moment) + pulled_job.args.as_mut().map(|args| { + args.insert("base_hash".to_owned(), to_raw_value(&*base_hash)) + }); + + let cloned_script = windmill_common::scripts::clone_script( + base_hash, + &pulled_job.workspace_id, + deployment_message, + &mut tx, + ) + .await?; + + if is_generated_from_raw_requirements(&Some(cloned_script.old_script.language), &cloned_script.old_script.lock.map(|v| v.to_string())) { + return Err(Error::BadRequest(format!( + "Script at path {} is generated from raw requirements, not overriding", + pulled_job.runnable_path() + ))); + } + + cloned_script.new_hash + } + JobKind::FlowDependencies => { + sqlx::query_scalar!( + "INSERT INTO flow_version + (workspace_id, path, value, schema, created_by) + + SELECT workspace_id, path, value, schema, created_by + FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3 + + RETURNING id + ", + pulled_job.runnable_path(), + pulled_job.workspace_id, + *base_hash, + ) + .fetch_one(&mut *tx) + .await? + } + JobKind::AppDependencies => { + sqlx::query_scalar!( + "INSERT INTO app_version + (app_id, value, created_by, raw_app) + SELECT app_id, value, created_by, raw_app + FROM app_version WHERE id = $1 + RETURNING id", + *base_hash + ) + .fetch_one(&mut *tx) + .await? + } + _ => { + return Err(Error::InternalErr(format!( + "Matched unexpected JobKind ({:?}). This is a bug!", + kind + ))) + } + }; + + pulled_job.runnable_id.replace(new_id.into()); + + if *windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING.read().await { + // === RETRIEVE ACCUMULATED DEBOUNCE DATA === + // + // For flows and apps, retrieve all nodes/components that were accumulated + // during the debounce window. This data comes from requests that were merged + // into this job instead of creating their own jobs. + // + // Scripts don't need this because they don't have nodes/components to relock. + if let Some(to_relock_field) = match &pulled_job.kind { + JobKind::FlowDependencies => Some("nodes_to_relock"), + JobKind::AppDependencies => Some("components_to_relock"), + _ => None, // Scripts don't use accumulated stale data + } { + tracing::debug!( + job_id = %pulled_job.id, + job_kind = ?pulled_job.kind, + field = %to_relock_field, + "Retrieving accumulated stale data from debounced requests" + ); + + if let Some(stale_data) = sqlx::query_scalar!( + "DELETE FROM debounce_stale_data WHERE job_id = $1 RETURNING to_relock", + &pulled_job.id + ) + .fetch_optional(&mut *tx) + .await + .map_err(|e| { + tracing::error!( + error = %e, + job_id = %pulled_job.id, + "Failed to retrieve debounce_stale_data" + ); + e + })? + .flatten() + { + tracing::debug!( + job_id = %pulled_job.id, + node_count = stale_data.len(), + nodes = ?stale_data, + "Retrieved accumulated nodes/components from {} debounced requests", + stale_data.len() + ); + + // Replace the job's relock list with the accumulated data + // This ensures all nodes from all debounced requests are processed + if let Some(args) = pulled_job.args.as_mut() { + args.insert(to_relock_field.to_owned(), to_raw_value(&stale_data)); + tracing::debug!( + field = %to_relock_field, + "Updated job args with accumulated debounce data" + ); + } + } else { + tracing::trace!( + job_id = %pulled_job.id, + "No accumulated stale data found (no debounced requests or already cleaned up)" + ); + } + } + } else { + tracing::warn!("Debouncing is not supported on this version of Windmill. Minimum version required for debouncing support."); + + } + // This will unblock pusher. + tx.commit().await?; + + Ok(()) + }).await; + } + + Ok(()) + } } /// Pull the job from queue @@ -2330,6 +2787,7 @@ pub async fn pull( job: None, suspended: false, missing_concurrency_key: false, + error_while_preprocessing: None, }); } @@ -2385,20 +2843,21 @@ pub async fn pull( // Concurrency limit is available for either enterprise job or dependency job && (cfg!(feature = "enterprise") || (job.is_dependency() && !*WMDEBUG_NO_DJOB_DEBOUNCING)) => { - let job = crate::jobs_ee::apply_concurrency_limit( + crate::jobs_ee::apply_concurrency_limit( db, pull_loop_count, suspended, job, ) - .await?; - job.unwrap_or(PulledJobResult { + .await? + .unwrap_or(PulledJobResult { job: None, suspended, missing_concurrency_key: false, + error_while_preprocessing: None, }) } - _ => PulledJobResult { job, suspended, missing_concurrency_key: false }, + _ => PulledJobResult { job, suspended, missing_concurrency_key: false, error_while_preprocessing: None }, }; Ok::<_, Error>(pulled_job_result) @@ -2416,7 +2875,7 @@ pub async fn pull( ) .await?; let Some(job) = job else { - return Ok(PulledJobResult { job: None, suspended, missing_concurrency_key: false }); + return Ok(PulledJobResult { job: None, suspended, missing_concurrency_key: false, error_while_preprocessing: None }); }; let has_concurent_limit = job.concurrent_limit.is_some(); @@ -2444,6 +2903,7 @@ pub async fn pull( job: Some(pulled_job), suspended, missing_concurrency_key: false, + error_while_preprocessing: None, }); } @@ -2451,11 +2911,11 @@ pub async fn pull( if cfg!(feature = "enterprise") || (pulled_job.is_dependency() && !*WMDEBUG_NO_DJOB_DEBOUNCING) { - if let Some(pulled_job) = + if let Some(pulled_job_res) = crate::jobs_ee::apply_concurrency_limit(db, pull_loop_count, suspended, pulled_job) .await? { - return Ok(pulled_job); + return Ok(pulled_job_res); } } } @@ -3123,33 +3583,29 @@ pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result( - id: Uuid, - w_id: &str, - tx: &mut Transaction<'c, Postgres>, -) -> error::Result> { - sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status - FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", - ) - .bind(id) - .bind(w_id) - .fetch_optional(&mut **tx) - .await - .map_err(Into::into) +pub fn get_mini_completed_job< +'a, +'e, +A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a, +>(id: &'a Uuid, w_id: &'a str, db: A) -> impl Future>> + Send + 'a { + async move { + let mut conn = db.acquire().await?; + sqlx::query_as!( + MiniCompletedJob, + "SELECT + j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as, + j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: JobTriggerKind\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl + FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id + WHERE j.id = $1 AND j.workspace_id = $2", + id, + w_id + ) + .fetch_optional(&mut *conn) + .await + .map_err(Into::into) + } } -pub async fn get_queued_job(id: &Uuid, w_id: &str, db: &DB) -> error::Result> { - sqlx::query_as::<_, QueuedJob>( - "SELECT *, null as workflow_as_code_status - FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", - ) - .bind(id) - .bind(w_id) - .fetch_optional(db) - .await - .map_err(Into::into) -} pub enum PushIsolationLevel<'c> { IsolatedRoot(DB), @@ -3405,6 +3861,7 @@ pub async fn push<'c, 'd>( mut email: &str, mut permissioned_as: String, token_prefix: Option<&str>, + #[allow(unused_mut)] mut scheduled_for_o: Option>, schedule_path: Option, parent_job: Option, @@ -3509,7 +3966,7 @@ pub async fn push<'c, 'd>( } let in_queue = sqlx::query_scalar!( - "SELECT COUNT(id) FROM v2_as_queue WHERE email = $1", + "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE j.permissioned_as_email = $1", email ) .fetch_one(_db) @@ -3523,7 +3980,7 @@ pub async fn push<'c, 'd>( } let concurrent_runs = sqlx::query_scalar!( - "SELECT COUNT(id) FROM v2_as_queue WHERE running = true AND email = $1", + "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE q.running = true AND j.permissioned_as_email = $1", email ) .fetch_one(_db) @@ -3607,6 +4064,7 @@ pub async fn push<'c, 'd>( } let mut preprocessed = None; + #[allow(unused)] let ( script_hash, script_path, @@ -4149,6 +4607,7 @@ pub async fn push<'c, 'd>( skip_expr: None, preprocessor_module: None, chat_input_enabled: None, + flow_env: None, notes: None, }; // this is a new flow being pushed, flow_status is set to flow_value: @@ -4724,11 +5183,6 @@ pub async fn push<'c, 'd>( ); } }; - #[cfg(not(all(feature = "enterprise", feature = "private")))] - { - let (_, _) = (debounce_delay_s, custom_debounce_key); - scheduled_for_o = scheduled_for_o; - } #[cfg(all(feature = "enterprise", feature = "private"))] if schedule_path.is_none() { @@ -5112,11 +5566,11 @@ async fn restarted_flows_resolution( > { let row = sqlx::query!( "SELECT - script_path, script_hash AS \"script_hash: ScriptHash\", - job_kind AS \"job_kind!: JobKind\", - flow_status AS \"flow_status: Json>\", - raw_flow AS \"raw_flow: Json>\" - FROM v2_as_completed_job WHERE id = $1 and workspace_id = $2", + j.runnable_path as script_path, j.runnable_id AS \"script_hash: ScriptHash\", + j.kind AS \"job_kind!: JobKind\", + COALESCE(c.flow_status, c.workflow_as_code_status) AS \"flow_status: Json>\", + j.raw_flow AS \"raw_flow: Json>\" + FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1 and j.workspace_id = $2", completed_flow_id, workspace_id, ) @@ -5372,240 +5826,3 @@ pub async fn get_same_worker_job( )) }) } - -pub async fn preprocess_dependency_job(job: &mut PulledJob, db: &DB) -> error::Result<()> { - let kind = job.kind; - // Handle dependency job debouncing cleanup when a job is pulled for execution - if kind.is_dependency() - && job - .args - .as_ref() - .map(|x| x.get("triggered_by_relative_import").is_some()) - .unwrap_or_default() - && !*WMDEBUG_NO_DJOB_DEBOUNCING - { - return Box::pin(async move { - // Only used for testing in tests/relative_imports.rs - // Give us some space to work with. - #[cfg(debug_assertions)] - if let Some(duration) = job - .args - .as_ref() - .map(|x| { - x.get("dbg_sleep_between_pull_and_debounce_key_removal") - .map(|v| serde_json::from_str::(v.get()).ok()) - .flatten() - }) - .flatten() - { - tracing::debug!("going to sleep",); - sleep(std::time::Duration::from_secs(duration as u64)).await; - } - - tracing::debug!( - "Processing debounce cleanup for dependency job {} at path {:?}", - &job.id, - &job.runnable_path - ); - - let key = format!("{}:{}:dependency", &job.workspace_id, job.runnable_path()); - let mut tx = db.begin().await?; - - // === DEBOUNCE CLEANUP === - // - // Clean up the debounce_key entry for this job (if it exists). - // - // IMPORTANT: We delete by key (not job_id) to avoid race conditions: - // If pusher has locked this row then this call will be blocked until all txs are commited. - // - // The idea is that the worker_lockfiles::trigger_dependents_to_recompute_locks will fetch the latest version of the obj. - // This object needs to be created before the djob is executed and it happens right here. - // - // This way the next pusher can fetch the latest version of object and base their djob payload on newest version. - // The concurrency limit on djobs will make sure that by the time next djob is started executing the base version it is referencing - // has already calculated all locks. This way even next djob will always use the fully finalized version of object. - // - // - // - // Note: We don't use a transaction here for performance (it's called during job pull). - // This means there's a tiny window where the job is running but key isn't deleted yet, - // which is acceptable because new requests will just accumulate data to this job. - tracing::debug!( - job_id = %job.id, - "Cleaning up debounce_key entry for completed/pulled job" - ); - - // This will either: - // 1. Block until pusher pushed. Which gives us: - // - If there was any stale data in pusher, then we will read it here (couple of lines below) - // 2. Block pusher until we are done here. This gives us: - // - We will clone objects and retrieve the latest version. So when we are done the pusher can read latest version. - sqlx::query!("DELETE FROM debounce_key WHERE key = $1", &key) - .execute(&mut *tx) - .await - .map_err(|e| { - tracing::error!( - error = %e, - job_id = %job.id, - "Failed to delete debounce_key" - ); - e - })?; - - let Some(base_hash) = job.runnable_id else { - return Err(Error::InternalErr( - "Missing runnable_id for dependency job triggered by relative import" - .to_string(), - )); - }; - - tracing::debug!( - job_id = %job.id, - base_hash = %base_hash, - job_kind = ?kind, - "Creating new version for dependency job triggered by relative import" - ); - - let new_id = match kind { - JobKind::Dependencies => { - let deployment_message = job - .args - .clone() - .map(|hashmap| { - hashmap - .get("deployment_message") - .map(|map_value| { - serde_json::from_str::(map_value.get()).ok() - }) - .flatten() - }) - .flatten(); - - // This way we tell downstream which script we should archive when the resolution is finished. - // (not used at the moment) - job.args.as_mut().map(|args| { - args.insert("base_hash".to_owned(), to_raw_value(&*base_hash)) - }); - - let new_hash = windmill_common::scripts::clone_script( - base_hash, - &job.workspace_id, - deployment_message, - &mut tx, - ) - .await?; - - new_hash - } - JobKind::FlowDependencies => { - sqlx::query_scalar!( - "INSERT INTO flow_version - (workspace_id, path, value, schema, created_by) - - SELECT workspace_id, path, value, schema, created_by - FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3 - - RETURNING id - ", - job.runnable_path(), - job.workspace_id, - *base_hash, - ) - .fetch_one(&mut *tx) - .await? - } - JobKind::AppDependencies => { - sqlx::query_scalar!( - "INSERT INTO app_version - (app_id, value, created_by, raw_app) - SELECT app_id, value, created_by, raw_app - FROM app_version WHERE id = $1 - RETURNING id", - *base_hash - ) - .fetch_one(&mut *tx) - .await? - } - _ => { - return Err(Error::InternalErr(format!( - "Matched unexpected JobKind ({:?}). This is a bug!", - kind - ))) - } - }; - - job.runnable_id.replace(new_id.into()); - - if !*windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING.read().await { - tx.commit().await?; - tracing::warn!("Debouncing is not supported on this version of Windmill. Minimum version required for debouncing support."); - return Ok(()); - } - // === RETRIEVE ACCUMULATED DEBOUNCE DATA === - // - // For flows and apps, retrieve all nodes/components that were accumulated - // during the debounce window. This data comes from requests that were merged - // into this job instead of creating their own jobs. - // - // Scripts don't need this because they don't have nodes/components to relock. - if let Some(to_relock_field) = match &job.kind { - JobKind::FlowDependencies => Some("nodes_to_relock"), - JobKind::AppDependencies => Some("components_to_relock"), - _ => None, // Scripts don't use accumulated stale data - } { - tracing::debug!( - job_id = %job.id, - job_kind = ?job.kind, - field = %to_relock_field, - "Retrieving accumulated stale data from debounced requests" - ); - - if let Some(stale_data) = sqlx::query_scalar!( - "DELETE FROM debounce_stale_data WHERE job_id = $1 RETURNING to_relock", - &job.id - ) - .fetch_optional(&mut *tx) - .await - .map_err(|e| { - tracing::error!( - error = %e, - job_id = %job.id, - "Failed to retrieve debounce_stale_data" - ); - e - })? - .flatten() - { - tracing::debug!( - job_id = %job.id, - node_count = stale_data.len(), - nodes = ?stale_data, - "Retrieved accumulated nodes/components from {} debounced requests", - stale_data.len() - ); - - // Replace the job's relock list with the accumulated data - // This ensures all nodes from all debounced requests are processed - if let Some(args) = job.args.as_mut() { - args.insert(to_relock_field.to_owned(), to_raw_value(&stale_data)); - tracing::debug!( - field = %to_relock_field, - "Updated job args with accumulated debounce data" - ); - } - } else { - tracing::trace!( - job_id = %job.id, - "No accumulated stale data found (no debounced requests or already cleaned up)" - ); - } - } - - // This will unblock pusher. - tx.commit().await?; - Ok(()) - }).await; - } - - Ok(()) -} diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index f36d2c5d58..d7e0f66685 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -115,6 +115,7 @@ nix.workspace = true bytes.workspace = true reqwest.workspace = true reqwest-middleware.workspace = true +eventsource-stream.workspace = true mime_guess.workspace = true hex.workspace = true tiberius = { workspace = true, optional = true } diff --git a/backend/windmill-worker/loader.bun.js b/backend/windmill-worker/loader.bun.js index 289fac0a9a..a697ba1d53 100644 --- a/backend/windmill-worker/loader.bun.js +++ b/backend/windmill-worker/loader.bun.js @@ -32,7 +32,12 @@ const p = { const imports = transpiler.scanImports(code); for (const imp of imports) { if (imp.kind == "import-statement") { - if (imp.path.startsWith(".") && !imp.path.endsWith(".ts")) { + if ( + (imp.path.startsWith(".") || + imp.path.startsWith("/u/") || + imp.path.startsWith("/f/")) && + !imp.path.endsWith(".ts") + ) { code = code.replaceAll(imp.path, imp.path + ".ts"); } } diff --git a/backend/windmill-worker/src/ai/providers/openai.rs b/backend/windmill-worker/src/ai/providers/openai.rs index 4fc4065766..d764cc11ad 100644 --- a/backend/windmill-worker/src/ai/providers/openai.rs +++ b/backend/windmill-worker/src/ai/providers/openai.rs @@ -8,6 +8,7 @@ use crate::ai::{ query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, sse::{OpenAISSEParser, SSEParser}, types::*, + utils::is_claude_model, }; // OpenAI-specific types @@ -79,6 +80,14 @@ pub struct OpenAIImageOutput { pub result: Option, // Base64 encoded image, None if not completed } +#[derive(Serialize, Debug)] +#[serde(rename_all = "lowercase")] +pub enum ToolChoice { + #[allow(dead_code)] + Auto, + Required, +} + #[derive(Serialize)] pub struct OpenAIRequest<'a> { pub model: &'a str, @@ -91,6 +100,8 @@ pub struct OpenAIRequest<'a> { pub max_completion_tokens: Option, #[serde(skip_serializing_if = "Option::is_none")] pub response_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, pub stream: bool, } @@ -192,6 +203,14 @@ impl OpenAIQueryBuilder { None }; + let is_claude_model = is_claude_model(&args.model); + // Force usage of structured output tool for Claude models when structured output provided + let tool_choice = if is_claude_model && response_format.is_some() { + Some(ToolChoice::Required) + } else { + None + }; + let request = OpenAIRequest { model: args.model, messages: &prepared_messages, @@ -199,6 +218,7 @@ impl OpenAIQueryBuilder { temperature: args.temperature, max_completion_tokens: args.max_tokens, response_format, + tool_choice, stream, }; diff --git a/backend/windmill-worker/src/ai/sse.rs b/backend/windmill-worker/src/ai/sse.rs index fcafc9cd7e..16cc418d52 100644 --- a/backend/windmill-worker/src/ai/sse.rs +++ b/backend/windmill-worker/src/ai/sse.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use eventsource_stream::Eventsource; use reqwest::Response; use serde::Deserialize; use serde_json; @@ -41,39 +42,29 @@ pub struct OpenAISSEEvent { pub choices: Option>, } +lazy_static::lazy_static! { + static ref DEBUG_SSE_STREAM: bool = std::env::var("DEBUG_SSE_STREAM") + .unwrap_or("false".to_string()) + .parse::() + .unwrap_or(false); +} pub trait SSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error>; async fn parse_events(&mut self, response: Response) -> Result<(), Error> { - let mut stream = response.bytes_stream(); - let mut buffer = String::new(); + let mut stream = response.bytes_stream().eventsource(); - while let Some(chunk_result) = stream.next().await { - let chunk = chunk_result - .map_err(|e| Error::internal_err(format!("Failed to read chunk: {}", e)))?; - - // Convert chunk to string and add to buffer - let chunk_str = String::from_utf8_lossy(&chunk); - buffer.push_str(&chunk_str); - - // Process complete lines from buffer - while let Some(newline_pos) = buffer.find("\n\n") { - let line = buffer.drain(..newline_pos + 2).collect::(); - let line = line.trim_end_matches('\n'); - - // Skip empty lines and comments - if line.is_empty() || line.starts_with(':') { - continue; - } - - // Parse SSE data field - if let Some(data) = line.strip_prefix("data: ") { - if data == "[DONE]" { - // OpenAI sends [DONE] to indicate end of stream - return Ok(()); + while let Some(event) = stream.next().await { + match event { + Ok(event) => { + if *DEBUG_SSE_STREAM { + tracing::info!("SSE event: {:?}", event); } - self.parse_event_data(data).await?; + self.parse_event_data(&event.data).await?; + } + Err(e) => { + tracing::error!("Failed to parse SSE event: {}", e); } } } @@ -102,50 +93,58 @@ impl OpenAISSEParser { impl SSEParser for OpenAISSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { - let event: OpenAISSEEvent = serde_json::from_str(data).map_err(|e| { - Error::internal_err(format!("Failed to parse SSE chunk {}: {}", data, e)) - })?; + if data == "[DONE]" { + return Ok(()); + } - if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { - if let Some(delta) = choices.remove(0).delta { - if let Some(content) = delta.content.filter(|s| !s.is_empty()) { - self.accumulated_content.push_str(&content); - let event = StreamingEvent::TokenDelta { content }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; - } + let event: Option = serde_json::from_str(data) + .inspect_err(|e| { + tracing::error!("Failed to parse SSE as an OpenAI event {}: {}", data, e); + }) + .ok(); - if let Some(tool_calls) = delta.tool_calls { - for (idx, tool_call) in tool_calls.into_iter().enumerate() { - let idx = tool_call.index.unwrap_or_else(|| idx as i64); + if let Some(event) = event { + if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { + if let Some(delta) = choices.remove(0).delta { + if let Some(content) = delta.content.filter(|s| !s.is_empty()) { + self.accumulated_content.push_str(&content); + let event = StreamingEvent::TokenDelta { content }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } - if let Some(function) = tool_call.function { - if let Some(tool_call) = self.accumulated_tool_calls.get_mut(&idx) { - if let Some(arguments) = function.arguments { - tool_call.function.arguments += &arguments; - } - } else { - let fun_name = function.name.unwrap_or_default(); - let call_id = tool_call.id.unwrap_or_else(|| rd_string(24)); - let event = StreamingEvent::ToolCall { - call_id: call_id.clone(), - function_name: fun_name.clone(), - }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; - self.accumulated_tool_calls.insert( - idx, - OpenAIToolCall { - id: call_id, - function: OpenAIFunction { - name: fun_name, - arguments: function.arguments.unwrap_or_default(), + if let Some(tool_calls) = delta.tool_calls { + for (idx, tool_call) in tool_calls.into_iter().enumerate() { + let idx = tool_call.index.unwrap_or_else(|| idx as i64); + + if let Some(function) = tool_call.function { + if let Some(tool_call) = self.accumulated_tool_calls.get_mut(&idx) { + if let Some(arguments) = function.arguments { + tool_call.function.arguments += &arguments; + } + } else { + let fun_name = function.name.unwrap_or_default(); + let call_id = tool_call.id.unwrap_or_else(|| rd_string(24)); + let event = StreamingEvent::ToolCall { + call_id: call_id.clone(), + function_name: fun_name.clone(), + }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + self.accumulated_tool_calls.insert( + idx, + OpenAIToolCall { + id: call_id, + function: OpenAIFunction { + name: fun_name, + arguments: function.arguments.unwrap_or_default(), + }, + r#type: "function".to_string(), }, - r#type: "function".to_string(), - }, - ); + ); + } } } } diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index fc45ed08cc..74f46c2bf4 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -6,7 +6,7 @@ use crate::ai::utils::{ update_flow_status_module_with_actions, update_flow_status_module_with_actions_success, FlowContext, }; -use crate::common::{error_to_value, OccupancyMetrics}; +use crate::common::OccupancyMetrics; use crate::result_processor::handle_non_flow_job_error; use crate::worker_flow::{ evaluate_input_transform, raw_script_to_payload, script_to_payload, JobPayloadWithTag, @@ -33,7 +33,8 @@ use windmill_common::{ worker::{to_raw_value, Connection}, }; use windmill_queue::{ - get_mini_pulled_job, push, JobCompleted, MiniPulledJob, PushArgs, PushIsolationLevel, + get_mini_pulled_job, push, JobCompleted, MiniCompletedJob, MiniPulledJob, PushArgs, + PushIsolationLevel, }; /// Context for tool execution containing all required references and state @@ -321,16 +322,23 @@ async fn execute_windmill_tool( // Evaluate each input transform and merge with AI-provided args for (key, transform) in input_transforms.iter() { // We skip static empty / null values, those are the one the AI will fill in - if let InputTransform::Static { value } = transform { - let val = value.get().trim(); - if val.is_empty() || val == "null" { + match transform { + InputTransform::Static { value } => { + let val = value.get().trim(); + if val.is_empty() || val == "null" { + continue; + } + } + InputTransform::Ai => { continue; } + _ => (), } let result = evaluate_input_transform::>( transform, last_result.clone(), flow_inputs.clone(), + None, Some(ctx.client), ctx.id_context.as_ref(), ) @@ -545,7 +553,7 @@ async fn execute_windmill_tool( ctx, tool_call, tool_module, - &tool_job, + &MiniCompletedJob::from(tool_job), job_id, err, messages, @@ -576,14 +584,14 @@ async fn handle_tool_execution_error( ctx: &mut ToolExecutionContext<'_>, tool_call: &OpenAIToolCall, tool_module: &windmill_common::flows::FlowModule, - tool_job: &MiniPulledJob, + tool_job: &MiniCompletedJob, job_id: Uuid, err: Error, messages: &mut Vec, final_events_str: &mut String, ) -> Result<(), Error> { let err_string = format!("{}: {}", err.name(), err.to_string()); - let err_json = error_to_value(&err); + let err_json = windmill_common::worker::error_to_value(&err); let _ = handle_non_flow_job_error( ctx.db, tool_job, diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index fb27c419ba..947b56f549 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -78,6 +78,7 @@ pub fn filter_schema_by_input_transforms( !val.is_empty() && val != "null" } InputTransform::Javascript { expr } => !expr.trim().is_empty(), + InputTransform::Ai => false, }; if is_completed { Some(key.clone()) diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index f9207e23a5..481717a207 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -405,7 +405,7 @@ pub async fn run_agent( .and_then(|fs| fs.memory_id) { // Read messages from memory - match read_from_memory(&job.workspace_id, memory_id, step_id).await { + match read_from_memory(db, &job.workspace_id, memory_id, step_id).await { Ok(Some(loaded_messages)) => { // Take the last n messages let start_idx = loaded_messages.len().saturating_sub(context_length); @@ -856,6 +856,7 @@ pub async fn run_agent( if let Some(memory_id) = flow_context.flow_status.and_then(|fs| fs.memory_id) { if let Err(e) = write_to_memory( + db, &job.workspace_id, memory_id, step_id, diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index cf93736850..c58b73f551 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -7,7 +7,7 @@ use serde_json::{json, value::RawValue, Value}; use windmill_common::client::AuthedClient; use windmill_common::error::to_anyhow; use windmill_common::s3_helpers::convert_json_line_stream; -use windmill_common::worker::Connection; +use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ parse_bigquery_sig, parse_db_resource, parse_s3_mode, parse_sql_blocks, @@ -86,9 +86,11 @@ fn do_bigquery_inner<'a>( timeout_ms: u64, column_order: Option<&'a mut Option>>, skip_collect: bool, + first_row_only: bool, http_client: &'a Client, s3: Option, -) -> windmill_common::error::Result>>> { +) -> windmill_common::error::Result>>>> +{ let param_names = parse_sql_statement_named_params(query, '@'); let statement_values = all_statement_values @@ -113,7 +115,7 @@ fn do_bigquery_inner<'a>( .json(&json!({ "query": query, "useLegacySql": false, - "maxResults": 10000, + "maxResults": if first_row_only { 1 } else { 10000 }, "timeoutMs": timeout_ms, "queryParameters": statement_values, })) @@ -126,7 +128,7 @@ fn do_bigquery_inner<'a>( match response.error_for_status_ref() { Ok(_) => { if skip_collect { - return Ok(to_raw_value(&Value::Array(vec![]))); + return Ok(vec![]); } else { let result = response.json::().await.map_err(|e| { Error::ExecutionErr(format!( @@ -205,10 +207,10 @@ fn do_bigquery_inner<'a>( convert_json_line_stream(rows_stream.boxed(), s3.format).await?; s3.upload(stream.boxed()).await?; - return Ok(to_raw_value(&s3.to_return_s3_obj())); + return Ok(vec![to_raw_value(&s3.to_return_s3_obj())]); } - Ok(to_raw_value(&rows)) + Ok(rows.iter().map(to_raw_value).collect::>()) } } Err(e) => match response.json::().await { @@ -341,6 +343,11 @@ pub async fn do_bigquery( }; let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = if annotations.return_last_result { + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; let service_account = CustomServiceAccount::from_json(&database) .map_err(|e| Error::ExecutionErr(e.to_string()))?; @@ -425,52 +432,34 @@ pub async fn do_bigquery( statement_values.insert(arg_n, bigquery_v); } - let result_f = if queries.len() > 1 { - let futures = queries - .iter() - .enumerate() - .map(|(i, x)| { - do_bigquery_inner( - x, - &statement_values, - &project_id, - token.as_str(), - timeout_ms, - None, - annotations.return_last_result && i < queries.len() - 1, - &http_client, - s3.clone(), - ) - }) - .collect::>>()?; + let result_f = async move { + let mut results = vec![]; + for (i, q) in queries.iter().enumerate() { + let result = do_bigquery_inner( + q, + &statement_values, + &project_id, + token.as_str(), + timeout_ms, + if i == queries.len() - 1 + && collection_strategy.collect_last_statement_only(queries.len()) + && !collection_strategy.collect_scalar() + { + Some(column_order) + } else { + None + }, + collection_strategy.collect_last_statement_only(queries.len()) + && i < queries.len() - 1, + collection_strategy.collect_first_row_only(), + &http_client, + s3.clone(), + )? + .await?; + results.push(result); + } - let f = async { - let mut res: Vec> = vec![]; - - for fut in futures { - let r = fut.await?; - res.push(r); - } - if annotations.return_last_result && res.len() > 0 { - Ok(res.pop().unwrap()) - } else { - Ok(to_raw_value(&res)) - } - }; - - f.boxed() - } else { - do_bigquery_inner( - query, - &statement_values, - &project_id, - token.as_str(), - timeout_ms, - Some(column_order), - false, - &http_client, - s3, - )? + collection_strategy.collect(results) }; let r = run_future_with_polling_update_job_poller( diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 4e03043c85..757656b934 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -313,6 +313,7 @@ pub async fn install_bun_lockfile( .env_clear() .envs(PROXY_ENVS.clone()) .envs(common_bun_proc_envs) + .envs(&*crate::worker::WHITELIST_ENVS) .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index bcc529258d..4a6edbd55d 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -34,7 +34,7 @@ use windmill_common::{ use anyhow::{anyhow, Result}; use windmill_parser_sql::{s3_mode_extension, S3ModeArgs, S3ModeFormat}; -use windmill_queue::MiniPulledJob; +use windmill_queue::{MiniCompletedJob, MiniPulledJob}; use std::collections::HashSet; use std::path::Path; @@ -539,13 +539,6 @@ pub async fn update_worker_ping_for_failed_init_script( } } -pub fn error_to_value(err: &Error) -> serde_json::Value { - match err { - Error::JsonErr(err) => err.clone(), - _ => json!({"message": err.to_string(), "name": err.name()}), - } -} - #[derive(Clone)] pub struct OccupancyMetrics { pub running_job_started_at: Option, @@ -973,7 +966,7 @@ pub async fn get_cached_resource_value_if_valid( pub async fn save_in_cache( db: &Pool, _client: &AuthedClient, - job: &MiniPulledJob, + job: &MiniCompletedJob, cached_path: String, r: Arc>, ) { @@ -1068,7 +1061,6 @@ pub fn build_http_client(timeout_duration: std::time::Duration) -> error::Result } pub fn get_root_job_id(job: &MiniPulledJob) -> uuid::Uuid { - // fallback to flow_innermost_root_job and parent_job as root_job is not set if equal to innermost root job or parent job job.root_job .or(job.flow_innermost_root_job) .or(job.parent_job) diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index 6ce2717d38..e65655c94d 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -76,7 +76,7 @@ pub async fn handle_dedicated_process( ) -> std::result::Result<(), error::Error> { //do not cache local dependencies - use windmill_queue::{JobCompleted, MiniPulledJob}; + use windmill_queue::{JobCompleted, MiniCompletedJob}; use crate::{handle_child::process_status, PROXY_ENVS}; let cmd_name = format!("dedicated {command_path}"); @@ -133,8 +133,7 @@ pub async fn handle_dedicated_process( } }); - let mut jobs: VecDeque> = - VecDeque::with_capacity(MAX_BUFFERED_DEDICATED_JOBS); + let mut jobs: VecDeque = VecDeque::with_capacity(MAX_BUFFERED_DEDICATED_JOBS); // let mut i = 0; // let mut j = 0; let mut alive = true; @@ -179,7 +178,7 @@ pub async fn handle_dedicated_process( } tracing::debug!("processed job: |{line}|"); if line.starts_with("wm_res[") { - let job: Arc = jobs.pop_front().expect("pop"); + let job = jobs.pop_front().expect("pop"); tracing::info!("job completed on dedicated worker {script_path}: {}", job.id); match serde_json::from_str::>(&line.replace("wm_res[success]:", "").replace("wm_res[error]:", "")) { Ok(result) => { @@ -215,12 +214,15 @@ pub async fn handle_dedicated_process( job = conditional_polling(jobs_rx.recv(), alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS) => { // i += 1; if let Some(job) = job { - jobs.push_back(job.clone()); - tracing::info!("received job and adding to queue on dedicated worker for {script_path}: {} (queue_size: {})", job.id, jobs.len()); + let id = job.id; + let args = serde_json::to_string(&job.args).expect("serialize"); + jobs.push_back(MiniCompletedJob::from(job)); + tracing::info!("received job and adding to queue on dedicated worker for {script_path}: {} (queue_size: {})", id, jobs.len()); // write_stdin(&mut stdin, &serde_json::to_string(&job.args.unwrap_or_else(|| serde_json::json!({"x": job.id}))).expect("serialize")).await?; - write_stdin(&mut stdin, &serde_json::to_string(&job.args).expect("serialize")).await?; + write_stdin(&mut stdin, &args).await?; stdin.flush().await.context("stdin flush")?; + // tracing::info!("wrote job to stdin for {script_path}: {} (queue_size: {})", id, jobs.len()); } else { tracing::debug!("job channel closed"); alive = false; diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index e3408ffca7..8acdbb8b07 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -1,6 +1,6 @@ use std::cell::RefCell; use std::env; -use std::ffi::{c_char, CStr, CString}; +use std::ffi::{c_char, c_uint, CStr, CString}; use std::ptr::NonNull; use std::sync::{Arc, Mutex}; @@ -12,7 +12,7 @@ use uuid::Uuid; use windmill_common::error::{to_anyhow, Error, Result}; use windmill_common::s3_helpers::{S3Object, S3_PROXY_LAST_ERRORS_CACHE}; use windmill_common::utils::sanitize_string_from_password; -use windmill_common::worker::Connection; +use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::workspaces::{get_ducklake_from_db_unchecked, DucklakeCatalogResourceType}; use windmill_parser_sql::{parse_duckdb_sig, parse_sql_blocks}; use windmill_queue::{CanceledBy, MiniPulledJob}; @@ -39,6 +39,20 @@ pub async fn do_duckdb( occupancy_metrics: &mut OccupancyMetrics, parent_runnable_path: Option, ) -> Result> { + let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = + if annotations.result_collection == SqlResultCollectionStrategy::Legacy { + // Before result_collection was introduced, duckdb ignored all statements results except the last one + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; + if annotations.return_last_result { + return Err(Error::ExecutionErr( + "return_last_result annotation is deprecated, use result_collection=last_statement_all_rows instead".to_string(), + )); + } + let token = client.token.clone(); let hidden_passwords = Arc::new(Mutex::new(Vec::::new())); @@ -141,11 +155,13 @@ pub async fn do_duckdb( &token, &base_internal_url, &w_id, + collection_strategy, ) }) .await .map_err(|e| Error::from(to_anyhow(e))) .and_then(|r| r); + let (result, column_order) = match result { Ok(r) => r, Err(e) => { @@ -210,6 +226,8 @@ struct DuckDbFfiLib { base_internal_url: *const c_char, w_id: *const c_char, column_order_ptr: *mut *mut c_char, + collect_last_only: bool, + collect_first_row_only: bool, ) -> *mut c_char, >, free_cstr: Symbol<'static, unsafe extern "C" fn(string: *mut c_char) -> ()>, @@ -249,7 +267,28 @@ impl DuckDbFfiLib { )) })? }; + let lib = Box::leak(Box::new(lib)); + + // Version mismatch should only be possible on Windows agent workers + // We check for it because FFI interface mismatch will cause undefined behavior / crashes + unsafe { + let expected_version: c_uint = 1; + let get_version: Symbol<'static, unsafe extern "C" fn() -> c_uint> = + lib.get(b"get_version") + .map_err(|e| return Error::ExecutionErr(format!("Could not find get_version in the duckdb ffi library. If you are not using docker, consider manually upgrading windmill_duckdb_ffi_lib. {}", e.to_string())))?; + let actual_version = get_version(); + if actual_version < expected_version { + return Err(Error::InternalErr( + format!("Incompatible duckdb ffi library version. Expected: {expected_version}, actual: {actual_version}. Please update to the latest windmill_duckdb_ffi_lib."), + )); + } else if actual_version > expected_version { + return Err(Error::InternalErr( + format!("Incompatible duckdb ffi library version. Expected: {expected_version}, actual: {actual_version}. Please upgrade your worker to the latest windmill version."), + )); + } + } + Ok(DuckDbFfiLib { run_duckdb_ffi: unsafe { lib.get(b"run_duckdb_ffi").map_err(to_anyhow)? }, free_cstr: unsafe { lib.get(b"free_cstr").map_err(to_anyhow)? }, @@ -265,6 +304,7 @@ fn run_duckdb_ffi_safe<'a>( token: &str, base_internal_url: &str, w_id: &str, + collection_strategy: SqlResultCollectionStrategy, ) -> Result<(Box, Option>)> { let query_block_list = query_block_list .map(|s| { @@ -296,13 +336,18 @@ fn run_duckdb_ffi_safe<'a>( base_internal_url.as_ptr(), w_id.as_ptr(), &mut column_order, + collection_strategy.collect_last_statement_only(query_block_list_count), + collection_strategy.collect_first_row_only(), ); let str = CStr::from_ptr(ptr).to_string_lossy().to_string(); free_cstr(ptr); str }; - let column_order = if column_order.is_null() { + let column_order = if column_order.is_null() + || !collection_strategy.collect_last_statement_only(query_block_list_count) + || collection_strategy.collect_scalar() + { None } else { let str = unsafe { CStr::from_ptr(column_order).to_string_lossy().to_string() }; @@ -313,7 +358,14 @@ fn run_duckdb_ffi_safe<'a>( if result_str.starts_with("ERROR") { Err(Error::ExecutionErr(result_str[6..].to_string())) } else { - let result = serde_json::value::RawValue::from_string(result_str).map_err(to_anyhow)?; + let result = if collection_strategy == SqlResultCollectionStrategy::AllStatementsAllRows { + // Avoid parsing JSON + serde_json::value::RawValue::from_string(result_str).map_err(to_anyhow)? + } else { + let result = + serde_json::from_str::>>>(&result_str).map_err(to_anyhow)?; + collection_strategy.collect(result)? + }; Ok((result, column_order)) } } @@ -350,7 +402,7 @@ fn format_attach_db_conn_str(db_resource: Value, db_type: &str) -> Result { let res: PgDatabase = serde_json::from_value(db_resource)?; format!( - "dbname={} {} host={} {} {}", + "dbname={} {} host={} {} {} {}", res.dbname, res.user.map(|u| format!("user={}", u)).unwrap_or_default(), res.host, @@ -358,6 +410,9 @@ fn format_attach_db_conn_str(db_resource: Value, db_type: &str) -> Result); +const FLOW_INPUT_PREFIX: &'static str = "flow_input"; +const ENV_KEY_PREFIX: &'static str = "flow_env"; +const DOT_PATTERN: &'static str = "."; +const START_BRACKET_PATTERN: &'static str = "[\""; +const END_BRACKET_PATTERN: &'static str = "\"]"; + +fn try_exact_property_access( + expr: &str, + flow_input: Option<&mappable_rc::Marc>>>, + flow_env: Option<&HashMap>>, +) -> Option> { + let obj = if expr.starts_with(FLOW_INPUT_PREFIX) { + Some(( + FLOW_INPUT_PREFIX, + flow_input.as_ref().map(|obj| obj.as_ref()), + )) + } else if expr.starts_with(ENV_KEY_PREFIX) { + Some((ENV_KEY_PREFIX, flow_env)) + } else { + None + }; + + if let Some((prefix, obj)) = obj { + let access_pattern_pos = prefix.len(); + let suffix = &expr[access_pattern_pos..]; + let maybe_key_name = if suffix.starts_with(DOT_PATTERN) { + let key_name_pos = DOT_PATTERN.len(); + Some(&suffix[key_name_pos..]) + } else if suffix.starts_with(START_BRACKET_PATTERN) { + let key_name_pos = START_BRACKET_PATTERN.len(); + let suffix = &suffix[key_name_pos..]; + + let flow_arg_name = suffix + .ends_with(END_BRACKET_PATTERN) + .then(|| { + let start_key_name_pos = access_pattern_pos + key_name_pos; + let end_key_name_pos = expr.len() - END_BRACKET_PATTERN.len(); + &expr[start_key_name_pos..end_key_name_pos] + }) + .filter(|s| s.len() > 0); + flow_arg_name + } else { + None + }; + + if let Some(key_name) = maybe_key_name { + if let Some(key_value) = obj.and_then(|obj| obj.get(key_name)) { + return Some(key_value.clone()); + } + } + } + None +} + +async fn handle_full_regex( + expr: &str, + authed_client: &AuthedClient, + by_id: &IdContext, +) -> Option>> { + if let Some(captures) = RE_FULL.captures(&expr) { + let obj_name = captures.get(1).unwrap().as_str(); + let obj_key = captures.get(2).unwrap().as_str(); + let idx_o = captures.get(3).map(|y| y.as_str()); + let rest = captures.get(4).map(|y| y.as_str()); + let query = if let Some(idx) = idx_o { + match rest { + Some(rest) => Some(format!("{}{}", idx, rest)), + None => Some(idx.to_string()), + } + } else { + rest.map(|x| x.trim_start_matches('.').to_string()) + }; + + let result = if obj_name == "results" { + authed_client + .get_result_by_id(&by_id.flow_job.to_string(), obj_key, query) + .await + } else if obj_name == "flow_env" { + authed_client + .get_flow_env_by_flow_job_id(&by_id.flow_job.to_string(), obj_key, query) + .await + } else { + unreachable!(); + }; + + return Some(result); + } + + return None; +} + pub async fn eval_timeout( expr: String, transform_context: HashMap>>, flow_input: Option>>>, + flow_env: Option<&HashMap>>, authed_client: Option<&AuthedClient>, by_id: Option<&IdContext>, #[allow(unused_variables)] ctx: Option>, @@ -180,21 +272,13 @@ pub async fn eval_timeout( expr, transform_context ); - for (k, v) in transform_context.iter() { - if k == &expr { - return Ok(v.as_ref().clone()); - } + + if let Some(value) = transform_context.get(&expr) { + return Ok(value.as_ref().to_owned()); } - if expr.starts_with("flow_input.") || expr.starts_with("flow_input[") { - if let Some(ref flow_input) = flow_input { - for (k, v) in flow_input.iter() { - if &format!("flow_input.{k}") == &expr || &format!("flow_input[\"{k}\"]") == &expr { - // tracing::error!("FLOW_INPUT"); - return Ok(v.clone()); - } - } - } + if let Some(value) = try_exact_property_access(&expr, flow_input.as_ref(), flow_env) { + return Ok(value); } let p_ids = by_id.map(|x| { @@ -219,24 +303,8 @@ pub async fn eval_timeout( } if let (Some(by_id), Some(authed_client)) = (by_id, authed_client) { - if let Some((id, idx_o, rest)) = RE_FULL.captures(&expr).map(|x| { - ( - x.get(1).unwrap().as_str(), - x.get(2).map(|y| y.as_str()), - x.get(3).map(|y| y.as_str()), - ) - }) { - let query = if let Some(idx) = idx_o { - match rest { - Some(rest) => Some(format!("{}{}", idx, rest)), - None => Some(idx.to_string()), - } - } else { - rest.map(|x| x.trim_start_matches('.').to_string()) - }; - return authed_client - .get_result_by_id(&by_id.flow_job.to_string(), id, query) - .await; + if let Some(result) = handle_full_regex(&expr, authed_client, by_id).await { + return result; } } @@ -271,8 +339,8 @@ pub async fn eval_timeout( if by_id.is_some() && authed_client.is_some() { ops.push(op_get_result()); ops.push(op_get_id()); + ops.push(op_get_flow_env()); } - let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() }; let exts = vec![ext]; // Use our snapshot to provision our new runtime @@ -307,13 +375,15 @@ pub async fn eval_timeout( let mut client = authed_client.clone(); if let Some(client) = client.as_mut() { client.force_client = Some( - configure_client(reqwest::ClientBuilder::new() - .user_agent("windmill/beta") - .danger_accept_invalid_certs( - std::env::var("ACCEPT_INVALID_CERTS").is_ok(), - )) - .build() - .unwrap(), + configure_client( + reqwest::ClientBuilder::new() + .user_agent("windmill/beta") + .danger_accept_invalid_certs( + std::env::var("ACCEPT_INVALID_CERTS").is_ok(), + ), + ) + .build() + .unwrap(), ); } op_state.put(OptAuthedClient(client)); @@ -323,7 +393,7 @@ pub async fn eval_timeout( .into_iter() .filter(|(a, _)| context_keys.contains(a)) .collect(), - }) + }); } sender @@ -378,11 +448,14 @@ fn replace_with_await(expr: String, fn_name: &str) -> String { s } lazy_static! { - static ref RE: Regex = - Regex::new(r#"(?m)(?Presults(?:\?)?(?:(?:\.[a-zA-Z_0-9]+)|(?:\[\".*?\"\])))"#).unwrap(); - static ref RE_FULL: Regex = - Regex::new(r"(?m)^results(?:\?)?\.([a-zA-Z_0-9]+)(?:\[(\d+)\])?((?:\.[a-zA-Z_0-9]+)+)?$") - .unwrap(); + static ref RE: Regex = Regex::new( + r#"(?m)(?P(?:results|flow_env)(?:\?)?(?:(?:\.[a-zA-Z_0-9]+)|(?:\[\".*?\"\])))"# + ) + .unwrap(); + static ref RE_FULL: Regex = Regex::new( + r"(?m)^(results|flow_env)(?:\?)?\.([a-zA-Z_0-9]+)(?:\[(\d+)\])?((?:\.[a-zA-Z_0-9]+)+)?$" + ) + .unwrap(); static ref RE_PROXY: Regex = Regex::new(r"^(https?)://(([^:@\s]+):([^:@\s]+)@)?([^:@\s]+)(:(\d+))?$").unwrap(); } @@ -453,6 +526,17 @@ const results = new Proxy({{}}, {{ }} }}); +async function flow_env_by_var_name(var_name) {{ + let root_job_id = "{}"; + return JSON.parse(await Deno.core.ops.op_get_flow_env(root_job_id, var_name, null)); +}} + +const flow_env = new Proxy({{}}, {{ + get: function(target, name, receiver) {{ + return flow_env_by_var_name(name); + }} +}}); + "#, by_id .steps_results @@ -469,6 +553,7 @@ const results = new Proxy({{}}, {{ .join(","), by_id.previous_id, by_id.flow_job, + by_id.flow_job ) } else { String::new() @@ -641,6 +726,33 @@ async fn op_resource( } } +#[cfg(feature = "deno_core")] +#[op2(async)] +#[string] +async fn op_get_flow_env( + op_state: Rc>, + #[string] root_job_id: String, + #[string] var_name: String, + #[string] json_path: Option, +) -> Result, deno_error::JsErrorBox> { + let client = op_state.borrow().borrow::().0.clone(); + if let Some(client) = client { + client + .get_flow_env_by_flow_job_id::>>( + &root_job_id, + &var_name, + json_path, + ) + .await + .map(|value| value.map(|val| val.get().to_string())) + .map_err(|e| deno_error::JsErrorBox::generic(e.to_string())) + } else { + Err(deno_error::JsErrorBox::generic( + "No client found in op state", + )) + } +} + #[cfg(feature = "deno_core")] pub struct TransformContext { pub envs: HashMap>>, @@ -1318,7 +1430,7 @@ multiline template`"; op_state.put(TransformContext { flow_input: None, envs: env.clone() }) } - let res = eval_timeout(code.to_string(), env, None, None, None, None).await?; + let res = eval_timeout(code.to_string(), env, None, None, None, None, None).await?; assert_eq!(res.get(), "2"); Ok(()) } diff --git a/backend/windmill-worker/src/memory_common.rs b/backend/windmill-worker/src/memory_common.rs index 16821c555b..440199702f 100644 --- a/backend/windmill-worker/src/memory_common.rs +++ b/backend/windmill-worker/src/memory_common.rs @@ -1,72 +1,123 @@ use crate::ai::types::OpenAIMessage; -use std::path::PathBuf; -use tokio::{fs, io::AsyncWriteExt}; use uuid::Uuid; -use windmill_common::worker::TMP_MEMORY_DIR; +use windmill_common::{db::DB, error::Error}; -/// Get the file path for storing memory for a specific AI agent step -pub fn path_for(workspace_id: &str, conversation_id: Uuid, step_id: &str) -> PathBuf { - PathBuf::from(TMP_MEMORY_DIR) - .join(workspace_id) - .join(conversation_id.to_string()) - .join(format!("{step_id}.json")) -} +pub const MAX_MEMORY_SIZE_BYTES: usize = 100_000; // 100KB per memory entry in database -/// Read messages from disk storage -pub async fn read_from_disk( +/// Read AI agent memory from database +pub async fn read_from_db( + db: &DB, workspace_id: &str, conversation_id: Uuid, step_id: &str, -) -> anyhow::Result>> { - let path = path_for(workspace_id, conversation_id, step_id); - if !fs::try_exists(&path).await? { - return Ok(None); - } +) -> Result>, Error> { + let result = sqlx::query!( + "SELECT messages FROM ai_agent_memory + WHERE workspace_id = $1 AND conversation_id = $2 AND step_id = $3", + workspace_id, + conversation_id, + step_id + ) + .fetch_optional(db) + .await?; - let bytes = fs::read(&path).await?; - let messages: Vec = serde_json::from_slice(&bytes)?; - Ok(Some(messages)) + match result { + Some(row) => { + let messages: Vec = serde_json::from_value(row.messages)?; + Ok(Some(messages)) + } + None => Ok(None), + } } -/// Write messages to disk storage -pub async fn write_to_disk( +/// Write AI agent memory to database with size checking and truncation +pub async fn write_to_db( + db: &DB, workspace_id: &str, conversation_id: Uuid, step_id: &str, messages: &[OpenAIMessage], -) -> anyhow::Result<()> { - let path = path_for(workspace_id, conversation_id, step_id); - - // Ensure parent directories exist - if let Some(dir) = path.parent() { - fs::create_dir_all(dir).await?; +) -> Result<(), Error> { + if messages.is_empty() { + return Ok(()); } - // Write atomically using a temporary file - let tmp = path.with_extension("json.tmp"); - let mut f = fs::File::create(&tmp).await?; - f.write_all(&serde_json::to_vec(messages)?).await?; - f.flush().await?; - drop(f); + // Serialize messages and check size + let mut messages_to_store = messages.to_vec(); + let mut json_value = serde_json::to_value(&messages_to_store)?; + let size_bytes = json_value.to_string().len(); - // Atomic rename - fs::rename(tmp, &path).await?; + // Truncate if necessary + if size_bytes > MAX_MEMORY_SIZE_BYTES { + tracing::warn!( + "Memory size ({} bytes) exceeds limit ({} bytes) for workspace={} conversation={} step={}. Truncating messages. Use S3 storage in workspace settings to store full conversation history.", + size_bytes, + MAX_MEMORY_SIZE_BYTES, + workspace_id, + conversation_id, + step_id + ); + + messages_to_store = truncate_messages(messages, MAX_MEMORY_SIZE_BYTES)?; + json_value = serde_json::to_value(&messages_to_store)?; + } + + // Insert or update using UPSERT + sqlx::query!( + "INSERT INTO ai_agent_memory (workspace_id, conversation_id, step_id, messages, created_at, updated_at) + VALUES ($1, $2, $3, $4, NOW(), NOW()) + ON CONFLICT (workspace_id, conversation_id, step_id) + DO UPDATE SET + messages = EXCLUDED.messages, + updated_at = NOW()", + workspace_id, + conversation_id, + step_id, + json_value + ) + .execute(db) + .await?; Ok(()) } -/// Delete all memory for a conversation from disk storage -pub async fn delete_conversation_from_disk( +/// Delete all memory for a conversation from database +pub async fn delete_conversation_from_db( + db: &DB, workspace_id: &str, conversation_id: Uuid, -) -> anyhow::Result<()> { - let conversation_path = PathBuf::from(TMP_MEMORY_DIR) - .join(workspace_id) - .join(conversation_id.to_string()); - - if fs::try_exists(&conversation_path).await? { - fs::remove_dir_all(&conversation_path).await?; - } +) -> Result<(), Error> { + sqlx::query!( + "DELETE FROM ai_agent_memory + WHERE workspace_id = $1 AND conversation_id = $2", + workspace_id, + conversation_id + ) + .execute(db) + .await?; Ok(()) } + +/// Truncate messages to fit within the size limit +fn truncate_messages( + messages: &[OpenAIMessage], + max_size_bytes: usize, +) -> Result, Error> { + let mut result = messages.to_vec(); + + // Keep removing oldest messages until we're under the threshold + while !result.is_empty() { + let test_json = serde_json::to_value(&result)?; + let test_size = test_json.to_string().len(); + + if test_size <= max_size_bytes { + break; + } + + // Remove the first (oldest) message + result.remove(0); + } + + Ok(result) +} diff --git a/backend/windmill-worker/src/memory_oss.rs b/backend/windmill-worker/src/memory_oss.rs index 2444cd0ebf..74771a79e8 100644 --- a/backend/windmill-worker/src/memory_oss.rs +++ b/backend/windmill-worker/src/memory_oss.rs @@ -3,23 +3,27 @@ pub use crate::memory_ee::*; #[cfg(not(all(feature = "private", feature = "enterprise")))] -use {crate::ai::types::OpenAIMessage, crate::memory_common, uuid::Uuid}; +use {crate::ai::types::OpenAIMessage, crate::memory_common, uuid::Uuid, windmill_common::db::DB}; /// Read AI agent memory from storage -/// In OSS: always reads from disk +/// In OSS: always reads from database #[cfg(not(all(feature = "private", feature = "enterprise")))] pub async fn read_from_memory( + db: &DB, workspace_id: &str, conversation_id: Uuid, step_id: &str, ) -> anyhow::Result>> { - memory_common::read_from_disk(workspace_id, conversation_id, step_id).await + memory_common::read_from_db(db, workspace_id, conversation_id, step_id) + .await + .map_err(|e| anyhow::anyhow!("Database read failed: {e:?}")) } /// Write AI agent memory to storage -/// In OSS: always writes to disk +/// In OSS: always writes to database #[cfg(not(all(feature = "private", feature = "enterprise")))] pub async fn write_to_memory( + db: &DB, workspace_id: &str, conversation_id: Uuid, step_id: &str, @@ -29,15 +33,20 @@ pub async fn write_to_memory( return Ok(()); } - memory_common::write_to_disk(workspace_id, conversation_id, step_id, messages).await + memory_common::write_to_db(db, workspace_id, conversation_id, step_id, messages) + .await + .map_err(|e| anyhow::anyhow!("Database write failed: {e:?}")) } /// Delete all memory for a conversation from storage -/// In OSS: always deletes from disk +/// In OSS: always deletes from database #[cfg(not(all(feature = "private", feature = "enterprise")))] pub async fn delete_conversation_memory( + db: &DB, workspace_id: &str, conversation_id: Uuid, ) -> anyhow::Result<()> { - memory_common::delete_conversation_from_disk(workspace_id, conversation_id).await + memory_common::delete_conversation_from_db(db, workspace_id, conversation_id) + .await + .map_err(|e| anyhow::anyhow!("Database delete failed: {e:?}")) } diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index c64e644be3..abb2d0087c 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -12,6 +12,7 @@ use tokio::net::TcpStream; use tokio_util::compat::TokioAsyncWriteCompatExt; use uuid::Uuid; use windmill_common::s3_helpers::convert_json_line_stream; +use windmill_common::worker::SqlResultCollectionStrategy; use windmill_common::{ error::{self, to_anyhow, Error}, utils::empty_as_none, @@ -95,6 +96,13 @@ pub async fn do_mssql( }; let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = if annotations.return_last_result { + SqlResultCollectionStrategy::LastStatementAllRows + } else if annotations.result_collection == SqlResultCollectionStrategy::Legacy { + SqlResultCollectionStrategy::AllStatementsAllRows + } else { + annotations.result_collection + }; let mut config = Config::new(); @@ -237,21 +245,20 @@ pub async fn do_mssql( let len = results.len(); let mut json_results = vec![]; for (i, statement_result) in results.into_iter().enumerate() { - if annotations.return_last_result && i < len - 1 { + if collection_strategy.collect_last_statement_only(len) && i < len - 1 { continue; } let mut json_rows = vec![]; for row in statement_result { - let row = row_to_json(row)?; + let row = to_raw_value(&row_to_json(row)?); json_rows.push(row); + if collection_strategy.collect_first_row_only() { + break; + } } json_results.push(json_rows); } - if annotations.return_last_result && json_results.len() > 0 { - Ok(to_raw_value(&json_results.pop().unwrap())) - } else { - Ok(to_raw_value(&json_results)) - } + collection_strategy.collect(json_results) } }; diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 55cbe35893..2163ac9890 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -16,7 +16,7 @@ use windmill_common::{ client::AuthedClient, error::{to_anyhow, Error}, s3_helpers::convert_json_line_stream, - worker::{to_raw_value, Connection}, + worker::{to_raw_value, Connection, SqlResultCollectionStrategy}, }; use windmill_parser_sql::{ parse_db_resource, parse_mysql_sig, parse_s3_mode, parse_sql_blocks, @@ -50,8 +50,10 @@ fn do_mysql_inner<'a>( conn: Arc>, column_order: Option<&'a mut Option>>, skip_collect: bool, + first_row_only: bool, s3: Option, -) -> windmill_common::error::Result>>> { +) -> windmill_common::error::Result>>>> +{ let param_names = parse_sql_statement_named_params(query, ':') .into_iter() .map(|x| x.into_bytes()) @@ -76,7 +78,7 @@ fn do_mysql_inner<'a>( .await .map_err(to_anyhow)?; - Ok(to_raw_value(&Value::Array(vec![]))) + Ok(vec![]) } else if let Some(ref s3) = s3 { let query = query.to_string(); let rows_stream = async_stream::stream! { @@ -108,14 +110,23 @@ fn do_mysql_inner<'a>( let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; s3.upload(stream.boxed()).await?; - Ok(to_raw_value(&s3.to_return_s3_obj())) + Ok(vec![to_raw_value(&s3.to_return_s3_obj())]) } else { - let rows: Vec = conn - .lock() - .await - .exec(query, statement_values) - .await - .map_err(to_anyhow)?; + let rows: Vec = if first_row_only { + conn.lock() + .await + .exec_first(query, statement_values) + .await + .map_err(to_anyhow)? + .into_iter() + .collect() + } else { + conn.lock() + .await + .exec(query, statement_values) + .await + .map_err(to_anyhow)? + }; if let Some(column_order) = column_order { *column_order = Some( @@ -130,12 +141,10 @@ fn do_mysql_inner<'a>( ); } - Ok(to_raw_value( - &rows - .into_iter() - .map(|x| convert_row_to_value(x)) - .collect::>(), - )) + Ok(rows + .into_iter() + .map(|x| to_raw_value(&convert_row_to_value(x))) + .collect::>()) } }; @@ -180,6 +189,11 @@ pub async fn do_mysql( }; let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = if annotations.return_last_result { + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; let opts = OptsBuilder::default() .db_name(Some(database.database)) @@ -285,45 +299,32 @@ pub async fn do_mysql( let queries = parse_sql_blocks(query); - let result_f = if queries.len() > 1 { - let futures = queries - .iter() - .enumerate() - .map(|(i, x)| { - do_mysql_inner( - x, - &statement_values, - conn_a.clone(), - None, - annotations.return_last_result && i < queries.len() - 1, - s3.clone(), - ) - }) - .collect::>>()?; + let conn_a_ref = &conn_a; + let result_f = async move { + let mut results = vec![]; + for (i, query) in queries.iter().enumerate() { + let result = do_mysql_inner( + query, + &statement_values, + conn_a_ref.clone(), + if i == queries.len() - 1 + && collection_strategy.collect_last_statement_only(queries.len()) + && !collection_strategy.collect_scalar() + { + Some(column_order) + } else { + None + }, + collection_strategy.collect_last_statement_only(queries.len()) + && i < queries.len() - 1, + collection_strategy.collect_first_row_only(), + s3.clone(), + )? + .await?; + results.push(result); + } - let f = async { - let mut res: Vec> = vec![]; - for fut in futures { - let r = fut.await?; - res.push(r); - } - if annotations.return_last_result && res.len() > 0 { - Ok(res.pop().unwrap()) - } else { - Ok(to_raw_value(&res)) - } - }; - - f.boxed() - } else { - do_mysql_inner( - query, - &statement_values, - conn_a.clone(), - Some(column_order), - false, - s3, - )? + collection_strategy.collect(results) }; let result = run_future_with_polling_update_job_poller( @@ -344,11 +345,10 @@ pub async fn do_mysql( pool.disconnect().await.map_err(to_anyhow)?; - let raw_result = windmill_common::worker::to_raw_value(&json!(result)); - *mem_peak = (raw_result.get().len() / 1000) as i32; + *mem_peak = (result.get().len() / 1000) as i32; // And then check that we got back the same string we sent over. - return Ok(raw_result); + return Ok(result); } // 2023-12-01T16:18:00.000Z diff --git a/backend/windmill-worker/src/oracledb_executor.rs b/backend/windmill-worker/src/oracledb_executor.rs index a1c20d1282..d83edf0104 100644 --- a/backend/windmill-worker/src/oracledb_executor.rs +++ b/backend/windmill-worker/src/oracledb_executor.rs @@ -11,7 +11,7 @@ use serde_json::{json, value::RawValue, Value}; use windmill_common::{ error::{to_anyhow, Error}, s3_helpers::convert_json_line_stream, - worker::{to_raw_value, Connection}, + worker::{to_raw_value, Connection, SqlResultCollectionStrategy}, }; use windmill_queue::MiniPulledJob; @@ -48,8 +48,10 @@ pub fn do_oracledb_inner<'a>( conn: Arc>, column_order: Option<&'a mut Option>>, skip_collect: bool, + first_row_only: bool, s3: Option, -) -> windmill_common::error::Result>>> { +) -> windmill_common::error::Result>>>> +{ let qw = query.trim_end_matches(';').to_string(); let result_f = async move { @@ -85,7 +87,7 @@ pub fn do_oracledb_inner<'a>( .map_err(to_anyhow)? .map_err(to_anyhow)?; - Ok(to_raw_value(&Value::Array(vec![]))) + Ok(vec![]) } else { // We use an mpsc because we need an async stream for s3 mode. However since everything is sync // in rust-oracle, I assumed that calling ResultSet::next() is blocking when it has to refetch. @@ -133,6 +135,9 @@ pub fn do_oracledb_inner<'a>( break; } } + if first_row_only { + break; + } } } _ => { @@ -159,17 +164,16 @@ pub fn do_oracledb_inner<'a>( if let Some(s3) = s3 { let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; s3.upload(stream.boxed()).await?; - return Ok(to_raw_value(&s3.to_return_s3_obj())); + return Ok(vec![to_raw_value(&s3.to_return_s3_obj())]); } else { let rows: Vec<_> = rows_stream.collect().await; - Ok(to_raw_value( - &rows - .into_iter() - .collect::, _>>() - .map_err(to_anyhow)? - .into_iter() - .collect::>(), - )) + Ok(rows + .into_iter() + .collect::, _>>() + .map_err(to_anyhow)? + .iter() + .map(to_raw_value) + .collect::>()) } } }; @@ -377,6 +381,11 @@ pub async fn do_oracledb( }; let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = if annotations.return_last_result { + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; let sig = parse_oracledb_sig(query) .map_err(|x| Error::ExecutionErr(x.to_string()))? @@ -388,7 +397,7 @@ pub async fn do_oracledb( let (query, args_to_skip) = sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args, &reserved_variables)?; - let (statement_values, errors) = get_statement_values(sig.clone(), &job_args, &args_to_skip); + let (_, errors) = get_statement_values(sig.clone(), &job_args, &args_to_skip); if !errors.is_empty() { return Err(Error::ExecutionErr(errors.join("\n"))); @@ -412,40 +421,33 @@ pub async fn do_oracledb( let queries = parse_sql_blocks(&query); - let result_f = if queries.len() > 1 { - let f = async { - let mut res: Vec> = vec![]; - for (i, q) in queries.iter().enumerate() { - let (vals, _) = get_statement_values(sig.clone(), &job_args, &args_to_skip); - let r = do_oracledb_inner( - q, - vals, - conn_a.clone(), - None, - annotations.return_last_result && i < queries.len() - 1, - s3.clone(), - )? - .await?; - res.push(r); - } + let result_f = async move { + let mut results = vec![]; + for (i, q) in queries.iter().enumerate() { + let (vals, _) = get_statement_values(sig.clone(), &job_args, &args_to_skip); - if annotations.return_last_result && res.len() > 0 { - Ok(res.pop().unwrap()) - } else { - Ok(to_raw_value(&res)) - } - }; + let result = do_oracledb_inner( + q, + vals, + conn_a.clone(), + if i == queries.len() - 1 + && collection_strategy.collect_last_statement_only(queries.len()) + && !collection_strategy.collect_scalar() + { + Some(column_order) + } else { + None + }, + collection_strategy.collect_last_statement_only(queries.len()) + && i < queries.len() - 1, + collection_strategy.collect_first_row_only(), + s3.clone(), + )? + .await?; + results.push(result); + } - f.boxed() - } else { - do_oracledb_inner( - &query, - statement_values, - conn_a, - Some(column_order), - false, - s3, - )? + collection_strategy.collect(results) }; let result = run_future_with_polling_update_job_poller( diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 9bb7c70cda..e240b5d87b 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -28,7 +28,9 @@ use uuid::Uuid; use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; use windmill_common::s3_helpers::convert_json_line_stream; -use windmill_common::worker::{to_raw_value, Connection, CLOUD_HOSTED}; +use windmill_common::worker::{ + to_raw_value, Connection, SqlResultCollectionStrategy, CLOUD_HOSTED, +}; use windmill_parser::{Arg, Typ}; use windmill_parser_sql::{ parse_db_resource, parse_pg_statement_arg_indices, parse_pgsql_sig, parse_s3_mode, @@ -73,8 +75,9 @@ fn do_postgresql_inner<'a>( column_order: Option<&'a mut Option>>, siz: &'a AtomicUsize, skip_collect: bool, + first_row_only: bool, s3: Option, -) -> error::Result>>> { +) -> error::Result>>>> { let mut query_params = vec![]; let arg_indices = parse_pg_statement_arg_indices(&query); @@ -100,7 +103,7 @@ fn do_postgresql_inner<'a>( let result_f = async move { // Now we can execute a simple statement that just returns its parameter. - let mut res: Vec = vec![]; + let mut res: Vec> = vec![]; let query_params = query_params .iter() @@ -125,13 +128,19 @@ fn do_postgresql_inner<'a>( let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; s3.upload(stream.boxed()).await?; - return Ok(to_raw_value(&s3.to_return_s3_obj())); + return Ok(vec![to_raw_value(&s3.to_return_s3_obj())]); } else { let rows = client .query_raw(&query, query_params) .await .map_err(to_anyhow)?; + let rows = if first_row_only { + rows.take(1).boxed() + } else { + rows.boxed() + }; + let rows = rows.try_collect::>().await.map_err(to_anyhow)?; if let Some(column_order) = column_order { @@ -164,14 +173,14 @@ fn do_postgresql_inner<'a>( } } if let Ok(v) = r { - res.push(v); + res.push(to_raw_value(&v)); } else { return Err(to_anyhow(r.err().unwrap()).into()); } } } - Ok(to_raw_value(&res)) + Ok(res) }; Ok(result_f.boxed()) @@ -216,6 +225,11 @@ pub async fn do_postgresql( }; let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = if annotations.return_last_result { + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; let sslmode = match database.sslmode.as_deref() { Some("allow") => "prefer".to_string(), @@ -336,47 +350,33 @@ pub async fn do_postgresql( .collect::>(); let size = AtomicUsize::new(0); - let result_f = if queries.len() > 1 { - let futures = queries - .iter() - .enumerate() - .map(|(i, x)| { - do_postgresql_inner( - x.to_string(), - ¶m_idx_to_arg_and_value, - client, - None, - &size, - annotations.return_last_result && i < queries.len() - 1, - s3.clone(), - ) - }) - .collect::>>()?; + let size_ref = &size; + let result_f = async move { + let mut results = vec![]; + for (i, query) in queries.iter().enumerate() { + let result = do_postgresql_inner( + query.to_string(), + ¶m_idx_to_arg_and_value, + client, + if i == queries.len() - 1 + && collection_strategy.collect_last_statement_only(queries.len()) + && !collection_strategy.collect_scalar() + { + Some(column_order) + } else { + None + }, + size_ref, + collection_strategy.collect_last_statement_only(queries.len()) + && i < queries.len() - 1, + collection_strategy.collect_first_row_only(), + s3.clone(), + )? + .await?; + results.push(result); + } - let f = async { - let mut res: Vec> = vec![]; - for fut in futures { - let r = fut.await?; - res.push(r); - } - if annotations.return_last_result && res.len() > 0 { - Ok(res.pop().unwrap()) - } else { - Ok(to_raw_value(&res)) - } - }; - - f.boxed() - } else { - do_postgresql_inner( - query.to_string(), - ¶m_idx_to_arg_and_value, - client, - Some(column_order), - &size, - false, - s3, - )? + collection_strategy.collect(results) }; let result = run_future_with_polling_update_job_poller( diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index cd3161ef12..4a4b85f8ac 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -426,9 +426,8 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", }\n"; // make sure param() is first - let param_match = windmill_parser_bash::RE_POWERSHELL_PARAM.find(&content); + let param_match = windmill_parser_bash::extract_powershell_param_block(&content, true); let content: String = if let Some(param_match) = param_match { - let param_match = param_match.as_str(); format!( "{}\n{}\n{}\n{}\n{}", param_match, diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 2c5b828313..b4b39a0c46 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -12,6 +12,7 @@ use tokio::{fs::DirBuilder, process::Command, sync::RwLock}; use uuid::Uuid; use windmill_common::{ error::{self, Error}, + lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, worker::Connection, }; @@ -22,7 +23,6 @@ use crate::{ common::{start_child_process, OccupancyMetrics}, handle_child::handle_child, python_executor::{PYTHON_PATH, UV_PATH}, - worker_lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, WIN_ENVS, }; diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 9f4fd7907d..1e168d3a30 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -22,7 +22,7 @@ use windmill_common::{ flow_status::FlowJobDuration, jobs::JobKind, utils::WarnAfterExt, - worker::{to_raw_value, Connection, WORKER_GROUP}, + worker::{error_to_value, to_raw_value, Connection, WORKER_GROUP}, worker_group_job_stats::{accumulate_job_stats, flush_stats_to_db, JobStatsMap}, KillpillSender, DB, }; @@ -31,8 +31,8 @@ use windmill_common::{ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ - append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, ValidableJson, - WrappedError, INIT_SCRIPT_TAG, + append_logs, get_mini_completed_job, CanceledBy, JobCompleted, MiniCompletedJob, MiniPulledJob, + ValidableJson, WrappedError, INIT_SCRIPT_TAG, }; use serde_json::{json, value::RawValue, Value}; @@ -43,7 +43,7 @@ use windmill_queue::{add_completed_job, add_completed_job_error}; use crate::{ bash_executor::ANSI_ESCAPE_RE, - common::{error_to_value, read_result, save_in_cache}, + common::{read_result, save_in_cache}, otel_oss::add_root_flow_job_to_otlp, worker_flow::update_flow_status_after_job_completion, JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult, SendResultPayload, @@ -400,7 +400,7 @@ async fn send_job_completed(job_completed_tx: JobCompletedSender, jc: JobComplet } pub async fn process_result( - job: Arc, + job: MiniCompletedJob, result: error::Result>>, job_dir: &str, job_completed_tx: JobCompletedSender, @@ -539,7 +539,7 @@ pub async fn handle_receive_completed_job( handle_job_error( db, &client, - job.as_ref(), + &job, mem_peak, canceled_by, err, @@ -721,7 +721,7 @@ pub async fn process_completed_job( pub async fn handle_non_flow_job_error( db: &DB, - job: &MiniPulledJob, + job: &MiniCompletedJob, mem_peak: i32, canceled_by: Option, err_string: String, @@ -752,7 +752,7 @@ pub async fn handle_non_flow_job_error( pub async fn handle_job_error( db: &DB, client: &AuthedClient, - job: &MiniPulledJob, + job: &MiniCompletedJob, mem_peak: i32, canceled_by: Option, err: Error, @@ -817,7 +817,7 @@ pub async fn handle_job_error( if let Err(err) = updated_flow { if let Some(parent_job_id) = job.parent_job { if let Ok(Some(parent_job)) = - get_queued_job(&parent_job_id, &job.workspace_id, &db).await + get_mini_completed_job(&parent_job_id, &job.workspace_id, db).await { let e = json!({"message": err.to_string(), "name": "InternalErr"}); append_logs( @@ -829,7 +829,7 @@ pub async fn handle_job_error( .await; let _ = add_completed_job_error( db, - &MiniPulledJob::from(&parent_job), + &parent_job, mem_peak, canceled_by.clone(), e, diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index 4dbc5bcf8a..cd25663889 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -891,9 +891,23 @@ fn wrap(inner_content: &str) -> Result { require 'json' a = JSON.parse(File.read("args.json")) -res = main(SPREAD) -File.open("result.json", "w") do |file| - file.write(JSON.generate(res)) + +begin + res = main(SPREAD) + File.open("result.json", "w") do |file| + file.write(JSON.generate(res)) + end + +rescue => e + error = { + name: e.class.name, + stack: e.full_message, + message: e.message + } + File.open("result.json", "w") do |file| + file.write(JSON.generate(error)) + end + raise end "# .replace("INNER_CONTENT", inner_content) diff --git a/backend/windmill-worker/src/scoped_dependency_map.rs b/backend/windmill-worker/src/scoped_dependency_map.rs index 09436bed3b..072fedc8a6 100644 --- a/backend/windmill-worker/src/scoped_dependency_map.rs +++ b/backend/windmill-worker/src/scoped_dependency_map.rs @@ -5,15 +5,12 @@ use windmill_common::{ cache, error::{Error, Result}, flows::{FlowModuleValue, FlowValue}, - scripts::ScriptLang, }; use std::collections::HashSet; -use crate::worker_lockfiles::{ - extract_relative_imports, is_generated_from_raw_requirements, - LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, -}; +use crate::worker_lockfiles::extract_relative_imports; +use windmill_common::lockfiles::is_generated_from_raw_requirements; // TODO: To be removed in future versions lazy_static::lazy_static! { @@ -295,16 +292,7 @@ SELECT importer_node_id, imported_path let mut dmap = ScopedDependencyMap::fetch(w_id, &r.path, "script", db).await?; let mut tx = db.begin().await?; - if (smd.language.is_some_and(|v| v == ScriptLang::Bun) - && sd - .lock - .as_ref() - .is_some_and(|v| v.contains("generatedFromPackageJson"))) - || (smd.language.is_some_and(|v| v == ScriptLang::Python3) - && sd.lock.as_ref().is_some_and(|v| { - v.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT) - })) - { + if is_generated_from_raw_requirements(&smd.language, &sd.lock) { // if the lock file is generated from a package.json/requirements.txt, we need to clear the dependency map // because we do not want to have dependencies be recomputed automatically. Empty relative imports passed // to update_script_dependency_map will clear the dependency map. @@ -350,7 +338,7 @@ SELECT importer_node_id, imported_path match fmv { // Since we fetched from flow_version it is safe to assume all inline scripts are in form of RawScript. FlowModuleValue::RawScript { content, language, lock ,.. } => { - if !is_generated_from_raw_requirements(Some(*language), lock) { + if !is_generated_from_raw_requirements(&Some(*language), lock) { to_process.push(( extract_relative_imports( content, diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index e657ef7bb0..38eff99ae1 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -10,7 +10,7 @@ use sha2::{Digest, Sha256}; use std::collections::HashMap; use windmill_common::error::to_anyhow; use windmill_common::s3_helpers::convert_json_line_stream; -use windmill_common::worker::Connection; +use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ @@ -130,10 +130,12 @@ fn do_snowflake_inner<'a>( token_is_keypair: bool, column_order: Option<&'a mut Option>>, skip_collect: bool, + first_row_only: bool, http_client: &'a Client, s3: Option, reserved_variables: &HashMap, -) -> windmill_common::error::Result>>> { +) -> windmill_common::error::Result>>>> +{ let sig = parse_snowflake_sig(&query) .map_err(|x| Error::ExecutionErr(x.to_string()))? .args; @@ -179,7 +181,7 @@ fn do_snowflake_inner<'a>( if skip_collect { handle_snowflake_result(result).await?; - Ok(to_raw_value(&Value::Array(vec![]))) + Ok(vec![]) } else { let response = result .parse_snowflake_response::() @@ -254,19 +256,22 @@ fn do_snowflake_inner<'a>( row_map }); + let rows_stream = rows_stream.take(if first_row_only { 1 } else { usize::MAX }); + if let Some(s3) = s3 { let rows_stream = rows_stream.map(|r| serde_json::value::to_value(&r?).map_err(to_anyhow)); let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; s3.upload(stream.boxed()).await?; - Ok(to_raw_value(&s3.to_return_s3_obj())) + Ok(vec![to_raw_value(&s3.to_return_s3_obj())]) } else { let rows = rows_stream .collect::>() .await .into_iter() + .map(|x| x.map(|v| to_raw_value(&v))) .collect::, _>>()?; - Ok(to_raw_value(&rows)) + Ok(rows) } } }; @@ -312,6 +317,11 @@ pub async fn do_snowflake( }; let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = if annotations.return_last_result { + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; // Check if the token is present in db_arg and use it if available let (token, token_is_keypair) = if let Some(token) = db_arg @@ -409,56 +419,38 @@ pub async fn do_snowflake( let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; - let result_f = if queries.len() > 1 { - let futures = queries - .iter() - .enumerate() - .map(|(i, x)| { - do_snowflake_inner( - x, - &snowflake_args, - body.clone(), - &database.account_identifier, - &token, - token_is_keypair, - None, - annotations.return_last_result && i < queries.len() - 1, - &http_client, - s3.clone(), - &reserved_variables, - ) - }) - .collect::>>()?; + let result_f = async move { + let mut results = vec![]; + for (i, q) in queries.iter().enumerate() { + let result = do_snowflake_inner( + q, + &snowflake_args, + body.clone(), + &database.account_identifier, + &token, + token_is_keypair, + if i == queries.len() - 1 + && collection_strategy.collect_last_statement_only(queries.len()) + && !collection_strategy.collect_scalar() + { + Some(column_order) + } else { + None + }, + collection_strategy.collect_last_statement_only(queries.len()) + && i < queries.len() - 1, + collection_strategy.collect_first_row_only(), + &http_client, + s3.clone(), + &reserved_variables, + )? + .await?; + results.push(result); + } - let f = async { - let mut res: Vec> = vec![]; - for fut in futures { - let r = fut.await?; - res.push(r); - } - if annotations.return_last_result && res.len() > 0 { - Ok(res.pop().unwrap()) - } else { - Ok(to_raw_value(&res)) - } - }; - - f.boxed() - } else { - do_snowflake_inner( - query, - &snowflake_args, - body.clone(), - &database.account_identifier, - &token, - token_is_keypair, - Some(column_order), - false, - &http_client, - s3.clone(), - &reserved_variables, - )? + collection_strategy.collect(results) }; + let r = run_future_with_polling_update_job_poller( job.id, job.timeout, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 0e6da36a20..e58d521a5d 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -18,6 +18,7 @@ use windmill_common::scripts::hash_to_codebase_id; use windmill_common::scripts::is_special_codebase_hash; use windmill_common::utils::report_critical_error; use windmill_common::utils::retrieve_common_worker_prefix; +use windmill_common::worker::error_to_value; use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, apps::AppScriptId, @@ -58,7 +59,7 @@ use std::{ time::Duration, }; use windmill_parser::MainArgSignature; -use windmill_queue::preprocess_dependency_job; +use windmill_queue::MiniCompletedJob; use windmill_queue::PulledJobResultToJobErr; use uuid::Uuid; @@ -111,7 +112,7 @@ use crate::{ bash_executor::handle_bash_job, bun_executor::handle_bun_job, common::{ - build_args_map, cached_result_path, error_to_value, get_cached_resource_value_if_valid, + build_args_map, cached_result_path, get_cached_resource_value_if_valid, get_reserved_variables, update_worker_ping_for_failed_init_script, OccupancyMetrics, }, csharp_executor::handle_csharp_job, @@ -326,6 +327,12 @@ lazy_static::lazy_static! { } proxy_env }; + pub static ref WHITELIST_ENVS: HashMap = { + windmill_common::worker::load_env_vars( + windmill_common::worker::load_whitelist_env_vars_from_env(), + &HashMap::new(), + ) + }; pub static ref DENO_PATH: String = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string()); pub static ref BUN_PATH: String = std::env::var("BUN_PATH").unwrap_or_else(|_| "/usr/bin/bun".to_string()); pub static ref NPM_PATH: String = std::env::var("NPM_PATH").unwrap_or_else(|_| "/usr/bin/npm".to_string()); @@ -741,7 +748,7 @@ fn create_span(arc_job: &Arc, worker_name: &str, hostname: &str) pub async fn handle_all_job_kind_error( conn: &Connection, authed_client: &AuthedClient, - job: Arc, + job: MiniCompletedJob, err: Error, same_worker_tx: Option<&SameWorkerSender>, worker_dir: &str, @@ -754,7 +761,7 @@ pub async fn handle_all_job_kind_error( handle_job_error( db, authed_client, - job.as_ref(), + &job, 0, None, err, @@ -773,7 +780,7 @@ pub async fn handle_all_job_kind_error( .send_job( JobCompleted { preprocessed_args: None, - job: job.clone(), + job: job, result: Arc::new(windmill_common::worker::to_raw_value(&error_to_value( &err, ))), @@ -834,10 +841,22 @@ pub fn start_interactive_worker_shell( match job { Ok(j) => match j.to_pulled_job() { - Ok(j) => Ok(j.map(NextJob::Sql)), - Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc)) => { - if let Err(err) = job_completed_tx.send_job(jc, true).await { - tracing::error!("An error occurred while sending job completed (missing concurrency key): {:#?}", err) + Ok(j) => Ok(j.clone().map(NextJob::Sql)), + ref e @ (Err(PulledJobResultToJobErr::MissingConcurrencyKey( + ref jc, + )) + | Err(PulledJobResultToJobErr::ErrorWhilePreprocessing( + ref jc, + ))) => { + if let Err(err) = + job_completed_tx.send_job(jc.clone(), true).await + { + let e_fmt = match e { + Ok(_) => "unknown error".to_owned(), + Err(e) => e.to_string(), + }; + + tracing::error!("An error occurred while sending job completed ({e_fmt}): {:#?}", err) } Ok(None) } @@ -954,8 +973,6 @@ pub async fn run_worker( killpill_tx: KillpillSender, base_internal_url: &str, ) { - - #[cfg(not(feature = "enterprise"))] if !*DISABLE_NSJAIL { tracing::warn!( @@ -1600,30 +1617,23 @@ pub async fn run_worker( } }; - // Essential debouncing job preprocessing. - if let Ok(windmill_queue::PulledJobResult { - job: Some(ref mut pulled_job), - .. - }) = &mut job - { - match timeout( + // Preprocess pulled job result + if let Ok(ref mut pulled_job_res) = job { + if let Err(e) = timeout( + // Will fail if longer than 10 seconds core::time::Duration::from_secs(10), - preprocess_dependency_job(pulled_job, &db), + pulled_job_res.preprocess(db), ) .warn_after_seconds(2) .await + // Flatten result + .map_err(error::Error::from) + .and_then(|r| r) { - Ok(Err(e)) => { - tracing::error!(worker = %worker_name, hostname = %hostname, "critical: debouncing job preprocessor failed: {e:?}"); - job = Err(e.into()); - } - Err(e) => { - tracing::error!(worker = %worker_name, hostname = %hostname, "critical: debouncing job preprocessor has timed out: {e:?}"); - job = Err(e.into()); - } - _ => {} + pulled_job_res.error_while_preprocessing = Some(e.to_string()); } } + add_time!(bench, "job pulled from DB"); let duration_pull_s = pull_time.elapsed().as_secs_f64(); let err_pull = job.is_ok(); @@ -1683,9 +1693,21 @@ pub async fn run_worker( match job { Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() { Ok(j) => Ok(j.map(NextJob::Sql)), - Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc)) => { - if let Err(err) = job_completed_tx.send_job(jc, true).await { - tracing::error!("An error occurred while sending job completed (missing concurrency key): {:#?}", err) + ref e @ (Err(PulledJobResultToJobErr::MissingConcurrencyKey( + ref jc, + )) + | Err(PulledJobResultToJobErr::ErrorWhilePreprocessing( + ref jc, + ))) => { + if let Err(err) = + job_completed_tx.send_job(jc.clone(), true).await + { + let e_fmt = match e { + Ok(_) => "unknown error".to_owned(), + Err(e) => e.to_string(), + }; + + tracing::error!("An error occurred while sending job completed ({e_fmt}): {:#?}", err) } Ok(None) } @@ -1749,7 +1771,7 @@ pub async fn run_worker( .send_job( JobCompleted { preprocessed_args: None, - job: Arc::new(job.job()), + job: MiniCompletedJob::from(job.job()), success: true, result: Arc::new(empty_result()), result_columns: None, @@ -1975,7 +1997,7 @@ pub async fn run_worker( handle_all_job_kind_error( &conn, &authed_client, - arc_job.clone(), + MiniCompletedJob::from(arc_job), err, Some(&same_worker_tx), &worker_dir, @@ -2364,7 +2386,6 @@ pub async fn handle_queued_job( precomputed_agent_info: Option, #[cfg(feature = "benchmark")] _bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { - // Extract the active span from the context if job.canceled_by.is_some() { @@ -2437,14 +2458,8 @@ pub async fn handle_queued_job( ) => { if x.map(|x| x.0).is_none_or(|x| is_special_codebase_hash(x)) { Some( - cache::job::fetch_preview( - conn, - &job.id, - raw_lock, - raw_code, - raw_flow.clone(), - ) - .await?, + cache::job::fetch_preview(conn, &job.id, raw_lock, raw_code, raw_flow.clone()) + .await?, ) } else { None @@ -2485,7 +2500,7 @@ pub async fn handle_queued_job( .send_job( JobCompleted { preprocessed_args: None, - job, + job: MiniCompletedJob::from(job), result, result_columns: None, mem_peak: 0, @@ -2506,10 +2521,7 @@ pub async fn handle_queued_job( tracing::debug!("Send job completed") } Err(err) => { - tracing::error!( - "An error occurred while sending job completed: {:#?}", - err - ) + tracing::error!("An error occurred while sending job completed: {:#?}", err) } } @@ -2544,7 +2556,6 @@ pub async fn handle_queued_job( )); } } else { - let mut logs = "".to_string(); let mut mem_peak: i32 = 0; let mut canceled_by: Option = None; @@ -2729,7 +2740,8 @@ pub async fn handle_queued_job( killpill_rx, precomputed_agent_info, &mut has_stream, - )).await; + )) + .await; occupancy_metrics.total_duration_of_running_jobs += metric_timer.elapsed().as_secs_f32(); @@ -2737,8 +2749,10 @@ pub async fn handle_queued_job( } }; + let cjob = MiniCompletedJob::from(job.to_owned()); + drop(job); //it's a test job, no need to update the db - if job.as_ref().workspace_id == "" { + if cjob.workspace_id == "" { return Ok(true); } @@ -2749,7 +2763,7 @@ pub async fn handle_queued_job( return Ok(false); } process_result( - job, + cjob, result.map(|x| Arc::new(x)), job_dir, job_completed_tx, @@ -2765,8 +2779,6 @@ pub async fn handle_queued_job( ) .await } - - } pub fn build_envs( @@ -2953,7 +2965,6 @@ async fn handle_code_execution_job( precomputed_agent_info: Option, has_stream: &mut bool, ) -> error::Result> { - let script_hash = || { job.runnable_id .ok_or_else(|| Error::internal_err("expected script hash")) @@ -2994,8 +3005,11 @@ async fn handle_code_execution_job( } JobKind::Script_Hub => { let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema } = - Box::pin(get_hub_script_content_and_requirements(job.runnable_path.as_ref(), conn.as_sql())) - .await?; + Box::pin(get_hub_script_content_and_requirements( + job.runnable_path.as_ref(), + conn.as_sql(), + )) + .await?; data = ScriptData { code: content, lock: lockfile }; metadata = ScriptMetadata { language, envs, codebase, schema, schema_validator: None }; @@ -3006,7 +3020,11 @@ async fn handle_code_execution_job( (arc_data.as_ref(), arc_metadata.as_ref()) } JobKind::FlowScript => { - arc_data = Box::pin(cache::flow::fetch_script(conn, FlowNodeId(script_hash()?.0))).await?; + arc_data = Box::pin(cache::flow::fetch_script( + conn, + FlowNodeId(script_hash()?.0), + )) + .await?; metadata = ScriptMetadata { language: job.script_lang, envs: None, @@ -3017,7 +3035,11 @@ async fn handle_code_execution_job( (arc_data.as_ref(), &metadata) } JobKind::AppScript => { - arc_data = Box::pin(cache::app::fetch_script(conn, AppScriptId(script_hash()?.0))).await?; + arc_data = Box::pin(cache::app::fetch_script( + conn, + AppScriptId(script_hash()?.0), + )) + .await?; metadata = ScriptMetadata { language: job.script_lang, envs: None, @@ -3035,8 +3057,11 @@ async fn handle_code_execution_job( .ok_or_else(|| Error::internal_err("expected script path".to_string()))?; if script_path.starts_with("hub/") { let ContentReqLangEnvs { content, lockfile, language, envs, codebase, schema } = - Box::pin(get_hub_script_content_and_requirements(Some(script_path), conn.as_sql())) - .await?; + Box::pin(get_hub_script_content_and_requirements( + Some(script_path), + conn.as_sql(), + )) + .await?; data = ScriptData { code: content, lock: lockfile }; metadata = ScriptMetadata { language, envs, codebase, schema, schema_validator: None }; @@ -3052,7 +3077,8 @@ async fn handle_code_execution_job( .await? .ok_or_else(|| Error::internal_err("expected script hash".to_string()))?; - (arc_data, arc_metadata) = Box::pin(cache::script::fetch(conn, ScriptHash(hash))).await?; + (arc_data, arc_metadata) = + Box::pin(cache::script::fetch(conn, ScriptHash(hash))).await?; (arc_data.as_ref(), arc_metadata.as_ref()) } } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 662c947842..f550bcd09d 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -36,6 +36,7 @@ use windmill_common::bench::BenchmarkIter; use windmill_common::cache::{self, RawData}; use windmill_common::client::AuthedClient; use windmill_common::db::Authed; +use windmill_common::flow_conversations::{add_message_to_conversation_tx, MessageType}; use windmill_common::flow_status::{ ApprovalConditions, FlowJobDuration, FlowJobsDuration, FlowStatusModuleWParent, Iterator as FlowIterator, JobResult, @@ -64,7 +65,7 @@ use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, handle_maybe_scheduled_job, insert_concurrency_key, interpolate_args, CanceledBy, - MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, + MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, }; use windmill_audit::audit_oss::{audit_log, AuditAuthor}; @@ -244,6 +245,7 @@ async fn evaluate_stop_after_all_iters_if( let stop_early_after_all_iters = compute_bool_from_expr( &stop_after_all_iters_if.expr, Marc::new(args), + None, iters_result.clone(), None, None, @@ -287,6 +289,11 @@ pub async fn update_flow_status_after_job_completion_internal( ) -> error::Result { let mut has_triggered_error_handler = has_triggered_error_handler; add_time!(bench, "update flow status internal START"); + struct ChatAiInfo { + chat_input_enabled: bool, + conversation_id: Option, + is_ai_agent_step: bool, + } let ( should_continue_flow, flow_job, @@ -296,6 +303,7 @@ pub async fn update_flow_status_after_job_completion_internal( nresult, is_failure_step, _cleanup_module, + chat_ai_info, ) = { // tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}"); @@ -420,17 +428,20 @@ pub async fn update_flow_status_after_job_completion_internal( ) .fetch_one(db) .await; + let args = + args.map(|flow_args| flow_args.map(|flow_args| flow_args.0).unwrap_or_default()); + args })); - let from_result_to_args = - |args: &Result>>>, sqlx::Error>| { - let args = args.as_ref().map_err(|e| { - Error::internal_err(format!("retrieval of args from state: {e:#}")) - })?; - Ok::<_, Error>(args.clone().unwrap_or_default().0) - }; + let from_result_to_args = |args: &Result>, sqlx::Error>| { + let args = args + .as_ref() + .map_err(|e| Error::internal_err(format!("retrieval of args from state: {e:#}")))?; + + Ok::<_, Error>(args.clone()) + }; let (mut stop_early, mut stop_early_err_msg, mut skip_if_stop_early, continue_on_error) = if stop_early_override.is_some() @@ -462,9 +473,11 @@ pub async fn update_flow_status_after_job_completion_internal( _ => None, }; let args = from_result_to_args(args.as_ref().await.get_ref())?; + compute_bool_from_expr( &expr, Marc::new(args), + None, result.clone(), all_iters, None, @@ -729,6 +742,7 @@ pub async fn update_flow_status_after_job_completion_internal( &mut stop_early_err_msg, &mut nresult, args, + ) .await?; } @@ -917,6 +931,7 @@ pub async fn update_flow_status_after_job_completion_internal( .and_then(|x| x.stop_after_all_iters_if.as_ref()) { let args = from_result_to_args(args.as_ref().await.get_ref())?; + evaluate_stop_after_all_iters_if( db, stop_after_all_iters_if, @@ -1009,6 +1024,7 @@ pub async fn update_flow_status_after_job_completion_internal( &old_status.retry, result.clone(), Marc::new(args), + None, Some(client), ) .await? @@ -1327,6 +1343,7 @@ pub async fn update_flow_status_after_job_completion_internal( &old_status.retry, result.clone(), Marc::new(args), + None, Some(client), ) .await? @@ -1359,6 +1376,11 @@ pub async fn update_flow_status_after_job_completion_internal( current_module_id = %current_module.map(|x| x.id.clone()).unwrap_or_default(), continue_on_error = %continue_on_error, should_continue_flow = %should_continue_flow, "computed if flow should continue"); + let chat_ai_info = ChatAiInfo { + chat_input_enabled: old_status.chat_input_enabled.unwrap_or(false), + conversation_id: old_status.memory_id, + is_ai_agent_step: current_module.is_some_and(|m| m.is_ai_agent()), + }; ( should_continue_flow, flow_job, @@ -1368,6 +1390,7 @@ pub async fn update_flow_status_after_job_completion_internal( nresult, is_failure_step, old_status.cleanup_module, + chat_ai_info, ) }; @@ -1437,37 +1460,61 @@ pub async fn update_flow_status_after_job_completion_internal( } if flow_job.is_canceled() { + let canceled_by = CanceledBy { + username: flow_job.canceled_by.clone(), + reason: flow_job.canceled_reason.clone(), + }; + let error = canceled_job_to_result(&flow_job); add_completed_job_error( db, - &flow_job, + &MiniCompletedJob::from(flow_job.clone()), 0, - Some(CanceledBy { - username: flow_job.canceled_by.clone(), - reason: flow_job.canceled_reason.clone(), - }), - canceled_job_to_result(&flow_job), + Some(canceled_by), + error, worker_name, true, None, ) .await?; } else { + let cflow_job: MiniCompletedJob = MiniCompletedJob::from(flow_job.clone()); + if flow_job.cache_ttl.is_some() && success { let flow = RawData::Flow(flow_data.clone()); let cached_res_path = cached_result_path(db, client, &flow_job, Some(&flow)).await; - save_in_cache(db, client, &flow_job, cached_res_path, nresult.clone()).await; + save_in_cache( + db, + client, + &MiniCompletedJob::from(cflow_job.clone()), + cached_res_path, + nresult.clone(), + ) + .await; } let success = success && (!is_failure_step || result_has_recover_true(nresult.clone())); add_time!(bench, "flow status update 1"); + + let skipped = stop_early && skip_if_stop_early; + add_tool_message_to_conversation( + db, + &job_id_for_status, + success, + skipped, + chat_ai_info.is_ai_agent_step, + &nresult, + chat_ai_info.chat_input_enabled, + chat_ai_info.conversation_id, + ) + .await?; let duration = if success { let (_, duration) = add_completed_job( db, - &flow_job, + &cflow_job, true, - stop_early && skip_if_stop_early, + skipped, Json(&nresult), None, 0, @@ -1482,9 +1529,9 @@ pub async fn update_flow_status_after_job_completion_internal( } else { let (_, duration) = add_completed_job( db, - &flow_job, + &cflow_job, false, - stop_early && skip_if_stop_early, + skipped, Json( &serde_json::from_str::(nresult.get()).unwrap_or_else( |e| json!({"error": format!("Impossible to serialize error: {e:#}")}), @@ -1531,8 +1578,17 @@ pub async fn update_flow_status_after_job_completion_internal( &db.into(), ) .await; - let _ = add_completed_job_error(db, &flow_job, 0, None, e, worker_name, true, None) - .await; + let _ = add_completed_job_error( + db, + &MiniCompletedJob::from(flow_job.clone()), + 0, + None, + e, + worker_name, + true, + None, + ) + .await; true } Ok(_) => false, @@ -1575,6 +1631,67 @@ fn find_flow_job_index(flow_jobs: &Vec, job_id_for_status: &Uuid) -> Optio flow_jobs.iter().position(|x| x == job_id_for_status) } +async fn add_tool_message_to_conversation( + db: &DB, + job_id: &Uuid, + success: bool, + skipped: bool, + is_ai_agent_step: bool, + result: &Box, + chat_input_enabled: bool, + conversation_id: Option, +) -> error::Result<()> { + // Create assistant message if it's a flow and it's done, but only if last module is not an AI agent + if !skipped && chat_input_enabled { + // Get conversation_id from flow_status.memory_id + + if let Some(conversation_id) = conversation_id { + // Only create assistant message if last module is NOT an AI agent, or there was an error + if !is_ai_agent_step || success == false { + let value = serde_json::to_value(result.get()) + .map_err(|e| Error::internal_err(format!("Failed to serialize result: {e}")))?; + + let content = match value { + // If it's an Object with "output" key AND the output is a String, return it + serde_json::Value::Object(mut map) + if map.contains_key("output") + && matches!(map.get("output"), Some(serde_json::Value::String(_))) => + { + if let Some(serde_json::Value::String(s)) = map.remove("output") { + s + } else { + // prettify the whole result + serde_json::to_string_pretty(&map) + .unwrap_or_else(|e| format!("Failed to serialize result: {e}")) + } + } + // Otherwise, if the whole value is a String, return it + serde_json::Value::String(s) => s, + // Otherwise, prettify the whole result + v => serde_json::to_string_pretty(&v) + .unwrap_or_else(|e| format!("Failed to serialize result: {e}")), + }; + + // Insert new assistant message + let mut tx = db.begin().await?; + add_message_to_conversation_tx( + &mut tx, + conversation_id, + Some(job_id.clone()), + &content, + MessageType::Assistant, + None, + success, + ) + .await?; + tx.commit().await?; + } + } + } + + Ok(()) +} + async fn set_success_and_duration_in_flow_job_success<'c>( flow_jobs_success: &Option>>, flow_jobs: &Vec, @@ -1717,6 +1834,7 @@ async fn evaluate_retry( status: &RetryStatus, result: Arc>, flow_args: Marc>>, + flow_env: Option<&HashMap>>, client: Option<&AuthedClient>, ) -> anyhow::Result> { if status.fail_count > MAX_RETRY_ATTEMPTS { @@ -1727,6 +1845,7 @@ async fn evaluate_retry( let should_retry = compute_bool_from_expr( &retry_if.expr, flow_args, + flow_env, result, None, None, @@ -1750,6 +1869,7 @@ async fn evaluate_retry( async fn compute_bool_from_expr( expr: &str, flow_args: Marc>>, + flow_env: Option<&HashMap>>, result: Arc>, all_iters: Option>>, by_id: Option<&IdContext>, @@ -1774,6 +1894,7 @@ async fn compute_bool_from_expr( format!("Boolean({expr})"), context, Some(flow_args), + flow_env, client, by_id, ctx, @@ -1799,15 +1920,13 @@ pub async fn evaluate_input_transform( transform: &InputTransform, last_result: Arc>, flow_args: Option>>>, + flow_env: Option<&HashMap>>, authed_client: Option<&AuthedClient>, by_id: Option<&IdContext>, ) -> error::Result where - T: for<'de> serde::Deserialize<'de> + Send, + T: for<'de> serde::Deserialize<'de> + Send + Default, { - let mut context = HashMap::with_capacity(2); - context.insert("result".to_string(), last_result.clone()); - context.insert("previous_result".to_string(), last_result.clone()); match transform { InputTransform::Static { value } => serde_json::from_str(value.get()).map_err(|e| { Error::ExecutionErr(format!( @@ -1816,10 +1935,14 @@ where )) }), InputTransform::Javascript { expr } => { + let mut context = HashMap::with_capacity(2); + context.insert("result".to_string(), last_result.clone()); + context.insert("previous_result".to_string(), last_result.clone()); let result = eval_timeout( expr.to_string(), context, flow_args, + flow_env, authed_client, by_id, None, @@ -1840,6 +1963,7 @@ where )) }) } + InputTransform::Ai => Ok(T::default()), } } @@ -1847,6 +1971,7 @@ where #[instrument(level = "trace", skip_all)] async fn transform_input( flow_args: Marc>>, + flow_env: Option<&HashMap>>, last_result: Arc>, input_transforms: &HashMap, resumes: Arc>, @@ -1888,6 +2013,7 @@ async fn transform_input( expr.to_string(), env.clone(), Some(flow_args.clone()), + flow_env, Some(client), Some(by_id), None, @@ -1900,6 +2026,7 @@ async fn transform_input( })?; mapped.insert(key.to_string(), v); } + InputTransform::Ai => (), } } @@ -1939,7 +2066,7 @@ pub async fn handle_flow( if let Some(schedule) = schedule { if let Err(err) = handle_maybe_scheduled_job( db, - &flow_job, + &MiniCompletedJob::from(flow_job.clone()), &schedule, flow_job.runnable_path.as_ref().unwrap(), &flow_job.workspace_id, @@ -1960,6 +2087,7 @@ pub async fn handle_flow( ); } } + let mut rec = PushNextFlowJobRec { flow_job: flow_job, status: status }; loop { let PushNextFlowJobRec { flow_job, status } = rec; @@ -2095,15 +2223,16 @@ async fn push_next_flow_job( // tracing::error!("status_module: {status_module:#?}"); let fj: mappable_rc::Marc = flow_job.clone().into(); - let arc_flow_job_args: Marc>> = Marc::map(fj, |x| { - if let Some(args) = &x.args { - &args.0 - } else { - &EHM - } - }); + let arc_flow_job_args: Marc>> = + Marc::map(fj, |x: &MiniPulledJob| { + if let Some(args) = &x.args { + &args.0 + } else { + &EHM + } + }); - // if this is an empty module without preprocessor of if the module has already been completed, successfully, update the parent flow + // if this is an empty module without preprocessor or if the module has already been completed, successfully, update the parent flow if (flow.modules.is_empty() && !step.is_preprocessor_step()) || matches!(status_module, FlowStatusModule::Success { .. }) { @@ -2191,6 +2320,7 @@ async fn push_next_flow_job( let skip = compute_bool_from_expr( &skip_expr, arc_flow_job_args.clone(), + flow.flow_env.as_ref(), Arc::new(to_raw_value(&json!("{}"))), None, None, @@ -2312,6 +2442,7 @@ async fn push_next_flow_job( expr.to_string(), context, Some(arc_flow_job_args.clone()), + flow.flow_env.as_ref(), None, None, None @@ -2334,6 +2465,9 @@ async fn push_next_flow_job( ))); } } + InputTransform::Ai => { + user_groups_required = Vec::new(); + } } } else { user_groups_required = Vec::new(); @@ -2569,6 +2703,7 @@ async fn push_next_flow_job( &input_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), + flow.flow_env.as_ref(), Some(client), None, ) @@ -2606,6 +2741,7 @@ async fn push_next_flow_job( &status.retry, arc_last_job_result.clone(), arc_flow_job_args.clone(), + flow.flow_env.as_ref(), Some(client), ) .await? @@ -2695,6 +2831,7 @@ async fn push_next_flow_job( compute_bool_from_expr( &skip_if.expr, arc_flow_job_args.clone(), + flow.flow_env.as_ref(), arc_last_job_result.clone(), None, Some(&idcontext), @@ -2786,6 +2923,7 @@ async fn push_next_flow_job( }; transform_input( arc_flow_job_args.clone(), + flow.flow_env.as_ref(), arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -2812,6 +2950,7 @@ async fn push_next_flow_job( let next_flow_transform = compute_next_flow_transform( arc_flow_job_args.clone(), arc_last_job_result.clone(), + flow.flow_env.as_ref(), &flow_job, &flow, transform_context, @@ -2960,6 +3099,7 @@ async fn push_next_flow_job( .await?; let ti = transform_input( Marc::new(args), + flow.flow_env.as_ref(), arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -3010,6 +3150,7 @@ async fn push_next_flow_job( .await?; let ti = transform_input( Marc::new(hm), + flow.flow_env.as_ref(), arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -3113,6 +3254,7 @@ async fn push_next_flow_job( timeout_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), + flow.flow_env.as_ref(), Some(client), Some(&ctx), ) @@ -3192,6 +3334,7 @@ async fn push_next_flow_job( parallelism_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), + flow.flow_env.as_ref(), Some(client), Some(&ctx), ) @@ -3574,11 +3717,13 @@ pub struct JobPayloadWithTag { pub timeout: Option, pub on_behalf_of: Option, } +#[derive(Debug)] enum ContinuePayload { SingleJob(JobPayloadWithTag), ParallelJobs(Vec), } +#[derive(Debug)] enum NextFlowTransform { EmptyInnerFlows { branch_chosen: Option }, Continue(ContinuePayload, NextStatus), @@ -3649,6 +3794,7 @@ pub fn get_path(flow_job: &MiniPulledJob, status: &FlowStatus, module: &FlowModu async fn compute_next_flow_transform( arc_flow_job_args: Marc>>, arc_last_job_result: Arc>, + flow_env: Option<&HashMap>>, flow_job: &MiniPulledJob, flow: &FlowValue, by_id: Option, @@ -3867,6 +4013,7 @@ async fn compute_next_flow_transform( resume, approvers, arc_flow_job_args, + flow_env, client, ¶llel, ) @@ -3962,6 +4109,7 @@ async fn compute_next_flow_transform( let pred = compute_bool_from_expr( &b.expr, arc_flow_job_args.clone(), + flow.flow_env.as_ref(), arc_last_job_result.clone(), None, Some(&idcontext), @@ -4226,6 +4374,7 @@ async fn next_forloop_status( resume: Arc>, approvers: Arc>, arc_flow_job_args: Marc>>, + flow_env: Option<&HashMap>>, client: &AuthedClient, parallel: &bool, ) -> Result { @@ -4241,6 +4390,11 @@ async fn next_forloop_status( /* Iterator is an InputTransform, evaluate it into an array. */ let itered_raw = match iterator { InputTransform::Static { value } => to_raw_value(value), + InputTransform::Ai => { + return Err(Error::ExecutionErr(format!( + "AI input transform not supported for iterator" + )))? + } InputTransform::Javascript { expr } => { let mut context = HashMap::with_capacity(5); context.insert("result".to_string(), arc_last_job_result.clone()); @@ -4253,6 +4407,7 @@ async fn next_forloop_status( expr.to_string(), context, Some(arc_flow_job_args), + flow_env, Some(client), Some(&by_id), None, @@ -4316,12 +4471,18 @@ async fn next_forloop_status( expr.to_string(), context, Some(arc_flow_job_args), + flow_env, Some(client), Some(&by_id), None, ) .await? } + InputTransform::Ai => { + return Err(Error::ExecutionErr(format!( + "AI input transform not supported for iterator" + )))? + } }; serde_json::from_str::>>(itered_raw.get()).map_err( |not_array| { diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 7b1acc19e8..44b68948a7 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -20,6 +20,7 @@ use windmill_common::error::Error; use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; use windmill_common::jobs::JobPayload; +use windmill_common::lockfiles::is_generated_from_raw_requirements; use windmill_common::scripts::ScriptHash; use windmill_common::utils::WarnAfterExt; #[cfg(feature = "python")] @@ -373,15 +374,7 @@ pub async fn process_relative_imports( db, ) .await?; - if (script_lang.is_some_and(|v| v == ScriptLang::Bun) - && lock - .as_ref() - .is_some_and(|v| v.contains("generatedFromPackageJson"))) - || (script_lang.is_some_and(|v| v == ScriptLang::Python3) - && lock - .as_ref() - .is_some_and(|v| v.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT))) - { + if is_generated_from_raw_requirements(script_lang, &lock) { // if the lock file is generated from a package.json/requirements.txt, we need to clear the dependency map // because we do not want to have dependencies be recomputed automatically. Empty relative imports passed // to update_script_dependency_map will clear the dependency map. @@ -445,17 +438,6 @@ pub async fn process_relative_imports( Ok(()) } -pub fn is_generated_from_raw_requirements(lang: Option, lock: &Option) -> bool { - (lang.is_some_and(|v| v == ScriptLang::Bun) - && lock - .as_ref() - .is_some_and(|v| v.contains("generatedFromPackageJson"))) - || (lang.is_some_and(|v| v == ScriptLang::Python3) - && lock - .as_ref() - .is_some_and(|v| v.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT))) -} - pub async fn trigger_dependents_to_recompute_dependencies( w_id: &str, script_path: &str, @@ -1450,7 +1432,7 @@ async fn lock_modules<'c>( if let Some(locks_to_reload) = locks_to_reload { if !locks_to_reload.contains(&e.id) { - if !is_generated_from_raw_requirements(Some(language), &lock) { + if !is_generated_from_raw_requirements(&Some(language), &lock) { let relative_imports = get_imports(); tx = dependency_map .patch(relative_imports.clone(), e.id.clone(), tx) @@ -1463,7 +1445,7 @@ async fn lock_modules<'c>( if lock.as_ref().is_some_and(|x| !x.trim().is_empty()) { let skip_creating_new_lock = skip_creating_new_lock(&language, &content); if skip_creating_new_lock { - if !is_generated_from_raw_requirements(Some(language), &lock) { + if !is_generated_from_raw_requirements(&Some(language), &lock) { let relative_imports = get_imports(); tx = dependency_map .patch(relative_imports.clone(), e.id.clone(), tx) @@ -2582,8 +2564,6 @@ async fn ansible_dep( serde_json::to_string(&ansible_lockfile).map_err(|e| e.into()) } -pub const LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT: &str = "# from requirements.txt"; - async fn capture_dependency_job( job_id: &Uuid, job_language: &ScriptLang, @@ -2672,7 +2652,11 @@ async fn capture_dependency_job( .await .map(|res| { if raw_deps { - format!("{}\n{}", LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, res) + format!( + "{}\n{}", + windmill_common::lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, + res + ) } else { res } diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 173a3f4480..4038cd90b3 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.571.0"; +export const VERSION = "v1.575.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 3903be6f71..1c95cb297d 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -180,6 +180,14 @@ export async function handleScriptMetadata( } } +export interface OutputFile { + path: string + contents: Uint8Array + hash: string + /** "contents" as text (changes automatically with "contents") */ + readonly text: string +} + export async function handleFile( path: string, workspace: Workspace, @@ -210,7 +218,9 @@ export async function handleFile( let bundleContent: string | Tarball | undefined = undefined; + let forceTar = false; if (codebase) { + let outputFiles: OutputFile[] = []; if (codebase.customBundler) { log.info(`Using custom bundler ${codebase.customBundler} for ${path}`); bundleContent = execSync( @@ -232,35 +242,42 @@ export async function handleFile( external: codebase.external, inject: codebase.inject, define: codebase.define, + loader: codebase.loader ?? { ".node": "file" }, + outdir: '/', platform: "node", packages: "bundle", target: format == "cjs" ? "node20.15.1" : "esnext", }); const endTime = performance.now(); bundleContent = out.outputFiles[0].text; + outputFiles = out.outputFiles; log.info( `Finished bundling ${path}: ${(bundleContent.length / 1024).toFixed( 0 )}kB (${(endTime - startTime).toFixed(0)}ms)` ); } - if (Array.isArray(codebase.assets) && codebase.assets.length > 0) { + if (outputFiles.length > 1) { const archiveNpm = await import("npm:@ayonli/jsext/archive"); log.info( - `Using the following asset configuration for ${path}: ${JSON.stringify( - codebase.assets - )}` + `Found multiple output files for ${path}, creating a tarball... ${outputFiles.map((file) => file.path).join(", ")}` ); + forceTar = true; const startTime = performance.now(); const tarball = new archiveNpm.Tarball(); + const mainPath = path.split(SEP).pop()?.split(".")[0] + ".js"; + const content = outputFiles.find((file) => file.path == "/" + mainPath)?.text ?? ''; + log.info(`Main content: ${content.length}chars`); tarball.append( - new File([bundleContent], "main.js", { type: "text/plain" }) + new File([content], "main.js", { type: "text/plain" }) ); - for (const asset of codebase.assets) { - const data = fs.readFileSync(asset.from); - const blob = new Blob([data], { type: "text/plain" }); - const file = new File([blob], asset.to); - tarball.append(file); + for (const file of outputFiles) { + if (file.path == "/" + mainPath) { + continue; + } + log.info(`Adding file: ${file.path.substring(1)}`); + const fil = new File([file.contents], file.path.substring(1)); + tarball.append(fil); } const endTime = performance.now(); log.info( @@ -269,6 +286,33 @@ export async function handleFile( ).toFixed(0)}kB (${(endTime - startTime).toFixed(0)}ms)` ); bundleContent = tarball; + } else { + if (Array.isArray(codebase.assets) && codebase.assets.length > 0) { + const archiveNpm = await import("npm:@ayonli/jsext/archive"); + log.info( + `Using the following asset configuration for ${path}: ${JSON.stringify( + codebase.assets + )}` + ); + const startTime = performance.now(); + const tarball = new archiveNpm.Tarball(); + tarball.append( + new File([bundleContent], "main.js", { type: "text/plain" }) + ); + for (const asset of codebase.assets) { + const data = fs.readFileSync(asset.from); + const blob = new Blob([data], { type: "text/plain" }); + const file = new File([blob], asset.to); + tarball.append(file); + } + const endTime = performance.now(); + log.info( + `Finished creating tarball for ${path}: ${( + tarball.size / 1024 + ).toFixed(0)}kB (${(endTime - startTime).toFixed(0)}ms)` + ); + bundleContent = tarball; + } } } let typed = opts?.skipScriptsMetadata @@ -325,7 +369,7 @@ export async function handleFile( } if (typed && codebase) { - typed.codebase = await codebase.getDigest(); + typed.codebase = await codebase.getDigest(forceTar); } const requestBodyCommon: NewScript = { @@ -352,7 +396,7 @@ export async function handleFile( concurrency_key: typed?.concurrency_key, debounce_key: typed?.debounce_key, debounce_delay_s: typed?.debounce_delay_s, - codebase: await codebase?.getDigest(), + codebase: await codebase?.getDigest(forceTar), timeout: typed?.timeout, on_behalf_of_email: typed?.on_behalf_of_email, }; diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 7733f02d8f..429fbd6ebc 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -4,7 +4,6 @@ import { colors, Command, Confirm, - Select, ensureDir, minimatch, JSZip, @@ -39,7 +38,6 @@ import { handleFile } from "../script/script.ts"; import { deepEqual, isFileResource } from "../../utils/utils.ts"; import { SyncOptions, - readConfigFile, getEffectiveSettings, validateBranchConfiguration, mergeConfigWithConfigFile, @@ -51,7 +49,6 @@ import { getBranchSpecificPath, fromBranchSpecificPath, isCurrentBranchFile, - toBranchSpecificPath, isBranchSpecificFile, } from "../../core/specific_items.ts"; import { getCurrentGitBranch } from "../../utils/git.ts"; @@ -1647,7 +1644,24 @@ function prettyChanges(changes: Change[], specificItems?: SpecificItemsConfig) { ) ); if (change.before != change.after) { - showDiff(change.before, change.after); + if (change.path.endsWith(".yaml")) { + try { + showDiff( + yamlStringify( + yamlParseContent(change.path, change.before), + yamlOptions + ), + yamlStringify( + yamlParseContent(change.path, change.after), + yamlOptions + ) + ); + } catch { + showDiff(change.before, change.after); + } + } else { + showDiff(change.before, change.after); + } } } } diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index e67617a562..cf74a1f4da 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -477,6 +477,7 @@ const command = new Command() .command("delete-fork") .description("Delete a forked workspace and git branch") .arguments("") + .option("-y --yes", "Skip confirmation prompt") .action(deleteWorkspaceFork as any); export default command; diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 5a56f97435..0a6dbe8167 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -99,6 +99,7 @@ export interface Codebase { external?: string[]; define?: { [key: string]: string }; inject?: string[]; + loader?: any, format?: "cjs" | "esm"; } diff --git a/cli/src/main.ts b/cli/src/main.ts index 146e5792f4..42d17a4231 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.571.0"; +export const VERSION = "1.575.0"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/cli/src/utils/codebase.ts b/cli/src/utils/codebase.ts index 36dfbb16ed..3391f6f795 100644 --- a/cli/src/utils/codebase.ts +++ b/cli/src/utils/codebase.ts @@ -2,7 +2,7 @@ import { Codebase, SyncOptions } from "../core/conf.ts"; import { log } from "../../deps.ts"; import { digestDir } from "./utils.ts"; -export type SyncCodebase = Codebase & { getDigest: () => Promise }; +export type SyncCodebase = Codebase & { getDigest: (forceTar?: boolean) => Promise }; export function listSyncCodebases( options: SyncOptions ): SyncCodebase[] { @@ -13,16 +13,21 @@ export function listSyncCodebases( } for (const codebase of options?.codebases ?? []) { let _digest: string | undefined = undefined; - const getDigest: () => Promise = async () => { - if (_digest == undefined) { + let alreadyPrinted = false; + const getDigest: (forceTar?: boolean) => Promise = async (forceTar?: boolean) => { + if (_digest == undefined || forceTar) { _digest = await digestDir( codebase.relative_path, JSON.stringify(codebase) ); - if (Array.isArray(codebase.assets) && codebase.assets.length > 0) { + if (forceTar || (Array.isArray(codebase.assets) && codebase.assets.length > 0)) { _digest += ".tar"; } - log.info(`Codebase ${codebase.relative_path}, digest: ${_digest}`); + if (!alreadyPrinted) { + alreadyPrinted = true; + log.info(`Codebase ${codebase.relative_path}, digest: ${_digest}`); + } + return _digest; } return _digest; }; diff --git a/cli/wasm/php/windmill_parser_wasm.d.ts b/cli/wasm/php/windmill_parser_wasm.d.ts index 7324082938..dcb548ab52 100644 --- a/cli/wasm/php/windmill_parser_wasm.d.ts +++ b/cli/wasm/php/windmill_parser_wasm.d.ts @@ -1,3 +1,3 @@ /* tslint:disable */ /* eslint-disable */ -export function parse_php(code: string): string; +export function parse_php(code: string, main_override?: string | null): string; diff --git a/cli/wasm/php/windmill_parser_wasm.js b/cli/wasm/php/windmill_parser_wasm.js index c0cdf9229d..e95e3a5126 100644 --- a/cli/wasm/php/windmill_parser_wasm.js +++ b/cli/wasm/php/windmill_parser_wasm.js @@ -56,30 +56,41 @@ function passStringToWasm0(arg, malloc, realloc) { return ptr; } -const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); +function isLikeNone(x) { + return x === undefined || x === null; +} + +let cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); }; +function decodeText(ptr, len) { + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + function getStringFromWasm0(ptr, len) { ptr = ptr >>> 0; - return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); + return decodeText(ptr, len); } /** * @param {string} code + * @param {string | null} [main_override] * @returns {string} */ -export function parse_php(code) { - let deferred2_0; - let deferred2_1; +export function parse_php(code, main_override) { + let deferred3_0; + let deferred3_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_php(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; + var ptr1 = isLikeNone(main_override) ? 0 : passStringToWasm0(main_override, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + var len1 = WASM_VECTOR_LEN; + const ret = wasm.parse_php(ptr0, len0, ptr1, len1); + deferred3_0 = ret[0]; + deferred3_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + wasm.__wbindgen_free(deferred3_0, deferred3_1, 1); } } diff --git a/cli/wasm/php/windmill_parser_wasm_bg.wasm b/cli/wasm/php/windmill_parser_wasm_bg.wasm index 2d772cc834..c92e10ebac 100644 Binary files a/cli/wasm/php/windmill_parser_wasm_bg.wasm and b/cli/wasm/php/windmill_parser_wasm_bg.wasm differ diff --git a/cli/wasm/php/windmill_parser_wasm_bg.wasm.d.ts b/cli/wasm/php/windmill_parser_wasm_bg.wasm.d.ts index a771d8a427..30c3d0eff5 100644 --- a/cli/wasm/php/windmill_parser_wasm_bg.wasm.d.ts +++ b/cli/wasm/php/windmill_parser_wasm_bg.wasm.d.ts @@ -1,7 +1,7 @@ /* tslint:disable */ /* eslint-disable */ export const memory: WebAssembly.Memory; -export const parse_php: (a: number, b: number) => [number, number]; +export const parse_php: (a: number, b: number, c: number, d: number) => [number, number]; export const __wbindgen_export_0: WebAssembly.Table; export const __wbindgen_malloc: (a: number, b: number) => number; export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; diff --git a/cli/wasm/regex/windmill_parser_wasm.d.ts b/cli/wasm/regex/windmill_parser_wasm.d.ts index 5ad7aef180..59f34f2b18 100644 --- a/cli/wasm/regex/windmill_parser_wasm.d.ts +++ b/cli/wasm/regex/windmill_parser_wasm.d.ts @@ -1,14 +1,14 @@ /* tslint:disable */ /* eslint-disable */ -export function parse_bash(code: string): string; -export function parse_snowflake(code: string): string; export function parse_assets_sql(code: string): string; -export function parse_duckdb(code: string): string; -export function parse_graphql(code: string): string; export function parse_db_resource(code: string): string | undefined; -export function parse_mysql(code: string): string; -export function parse_sql(code: string): string; -export function parse_oracledb(code: string): string; -export function parse_bigquery(code: string): string; +export function parse_bash(code: string): string; export function parse_mssql(code: string): string; +export function parse_oracledb(code: string): string; export function parse_powershell(code: string): string; +export function parse_bigquery(code: string): string; +export function parse_graphql(code: string): string; +export function parse_snowflake(code: string): string; +export function parse_duckdb(code: string): string; +export function parse_sql(code: string): string; +export function parse_mysql(code: string): string; diff --git a/cli/wasm/regex/windmill_parser_wasm.js b/cli/wasm/regex/windmill_parser_wasm.js index ce0a45c4c4..25a60a9b8e 100644 --- a/cli/wasm/regex/windmill_parser_wasm.js +++ b/cli/wasm/regex/windmill_parser_wasm.js @@ -68,44 +68,6 @@ function getStringFromWasm0(ptr, len) { ptr = ptr >>> 0; return decodeText(ptr, len); } -/** - * @param {string} code - * @returns {string} - */ -export function parse_bash(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_bash(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_snowflake(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_snowflake(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - /** * @param {string} code * @returns {string} @@ -125,44 +87,6 @@ export function parse_assets_sql(code) { } } -/** - * @param {string} code - * @returns {string} - */ -export function parse_duckdb(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_duckdb(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_graphql(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_graphql(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - /** * @param {string} code * @returns {string | undefined} @@ -183,70 +107,13 @@ export function parse_db_resource(code) { * @param {string} code * @returns {string} */ -export function parse_mysql(code) { +export function parse_bash(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_mysql(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_sql(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_sql(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_oracledb(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_oracledb(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_bigquery(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_bigquery(ptr0, len0); + const ret = wasm.parse_bash(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -274,6 +141,25 @@ export function parse_mssql(code) { } } +/** + * @param {string} code + * @returns {string} + */ +export function parse_oracledb(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_oracledb(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + /** * @param {string} code * @returns {string} @@ -293,6 +179,120 @@ export function parse_powershell(code) { } } +/** + * @param {string} code + * @returns {string} + */ +export function parse_bigquery(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_bigquery(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_graphql(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_graphql(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_snowflake(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_snowflake(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_duckdb(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_duckdb(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_sql(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_sql(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_mysql(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_mysql(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + const imports = { __wbindgen_placeholder__: { __wbindgen_init_externref_table: function() { diff --git a/cli/wasm/regex/windmill_parser_wasm_bg.wasm b/cli/wasm/regex/windmill_parser_wasm_bg.wasm index 12a3a0f992..45a144132d 100644 Binary files a/cli/wasm/regex/windmill_parser_wasm_bg.wasm and b/cli/wasm/regex/windmill_parser_wasm_bg.wasm differ diff --git a/docker-compose.yml b/docker-compose.yml index 1770a1e4db..ea67377ffc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,8 +73,6 @@ services: - /var/run/docker.sock:/var/run/docker.sock - worker_dependency_cache:/tmp/windmill/cache - worker_logs:/tmp/windmill/logs - # for AI agent memory - - worker_memory:/tmp/windmill/memory logging: *default-logging diff --git a/frontend/README.md b/frontend/README.md index 850df24593..377a4e0ed9 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,5 +1,5 @@ # Windmill frontend -The Windmill frontend written in Svelte + Tailwind CSS +The Windmill frontend written in Svelte 5 + Tailwind CSS The frontend is under AGPL, see the LICENSE file at the root of this repo diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 61855e46cc..d651c99a6f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.571.0", + "version": "1.575.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.571.0", + "version": "1.575.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -77,9 +77,9 @@ "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", - "windmill-parser-wasm-php": "1.510.1", + "windmill-parser-wasm-php": "1.574.1", "windmill-parser-wasm-py": "1.538.0", - "windmill-parser-wasm-regex": "1.565.0", + "windmill-parser-wasm-regex": "1.574.1", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.565.0", @@ -13749,9 +13749,9 @@ "integrity": "sha512-AJLFiUy6af+LpUe7CddDo4+JOmw3c0K/1iOWh8NdTwXcLDj90lL6089mdsVo1apyloLgrTbcuFDzZMXVGBgtCg==" }, "node_modules/windmill-parser-wasm-php": { - "version": "1.510.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-php/-/windmill-parser-wasm-php-1.510.1.tgz", - "integrity": "sha512-qM+yeaqPdMuAaPpqND31ZabpeHlPxxtmRWLs11cGHOCHU32FIaZ92/icNUoAxgBhdJjRhR4GXaeZG32nWit2cg==" + "version": "1.574.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-php/-/windmill-parser-wasm-php-1.574.1.tgz", + "integrity": "sha512-COyid6B1RYs+bpzUCInsA4HY/WZkpDLfkQ90+AqU/TVTpzYSbAC2JCbIwy0cRElBvlhI4bQ+9Wg6hSQKMpEkpA==" }, "node_modules/windmill-parser-wasm-py": { "version": "1.538.0", @@ -13759,9 +13759,9 @@ "integrity": "sha512-s+bdIgT/fA5em3zYUwF8D14uA/dZh7iu0krZYZQqZUO7txN37hwSCVfovbMkIwm4zPbsJ50mU8DRLt7UpAPZIw==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.565.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.565.0.tgz", - "integrity": "sha512-fcAy7SkyrCML6YgnsQ5H3utpJFxLqWssGmY+hJAibdQtAYNRASsSQZl87A46nIFO8TpKKiWioTSxZDl/9Ovtrw==" + "version": "1.574.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.574.1.tgz", + "integrity": "sha512-KnNBnpTGcnBPSzTQHdsjTPJbRQ81iem98eUnMGO8Zt1qgBFJ/eFZPwmUg6TZMbMITzTi0QqRCJj08eQID1FcPg==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", diff --git a/frontend/package.json b/frontend/package.json index 3b10f64bc8..dfb0fda59a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.571.0", + "version": "1.575.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -142,9 +142,9 @@ "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", - "windmill-parser-wasm-php": "1.510.1", + "windmill-parser-wasm-php": "1.574.1", "windmill-parser-wasm-py": "1.538.0", - "windmill-parser-wasm-regex": "1.565.0", + "windmill-parser-wasm-regex": "1.574.1", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.565.0", diff --git a/frontend/src/lib/common.ts b/frontend/src/lib/common.ts index e5220a2655..5f30b6056e 100644 --- a/frontend/src/lib/common.ts +++ b/frontend/src/lib/common.ts @@ -50,6 +50,7 @@ export interface SchemaProperty { oneOf?: SchemaProperty[] originalType?: string disabled?: boolean + 'x-no-s3-storage-workspace-warning'?: string } export interface ModalSchemaProperty { diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 1c631b0d9b..b63d3798e5 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -14,7 +14,7 @@ import { DollarSign, Plus, X, Check, Loader2, ExternalLink } from 'lucide-svelte' import { createEventDispatcher, onDestroy, onMount, tick, untrack } from 'svelte' import { fade } from 'svelte/transition' - import { Button, SecondsInput } from './common' + import { Alert, Button, SecondsInput } from './common' import FieldHeader from './FieldHeader.svelte' import type ItemPicker from './ItemPicker.svelte' import ObjectResourceInput from './ObjectResourceInput.svelte' @@ -126,6 +126,7 @@ | undefined) | undefined workspace?: string | undefined + s3StorageConfigured?: boolean actions?: import('svelte').Snippet innerBottomSnippet?: import('svelte').Snippet fieldHeaderActions?: import('svelte').Snippet @@ -185,6 +186,7 @@ appPath = undefined, computeS3ForceViewerPolicies = undefined, workspace = undefined, + s3StorageConfigured = true, actions, innerBottomSnippet, fieldHeaderActions, @@ -398,6 +400,7 @@ const UUID_PATTERN = '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' const IPV6_PATTERN = '^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$' + function validateInput(pattern: string | undefined, v: any, required: boolean): void { if (nullable && emptyString(v)) { error = '' @@ -405,6 +408,18 @@ } else if (required && (v == undefined || v == null || v === '') && inputCat != 'object') { error = 'Required' valid && (valid = false) + } else if ( + required && + inputCat == 'list' && + extra?.['nonEmpty'] == true && + Array.isArray(v) && + v.length === 0 + ) { + error = 'Required' + valid && (valid = false) + } else if (inputCat == 'list' && !Array.isArray(v)) { + error = 'Expected an array, got ' + typeof v + ' instead' + valid && (valid = false) } else { if (inputCat == 'number' && typeof v === 'number') { let min = extra['min'] @@ -521,6 +536,7 @@ }) $effect(() => { + extra?.['nonEmpty'] let args = [pattern, value, required] as const untrack(() => validateInput(...args)) }) @@ -1455,6 +1471,15 @@ {:else if !noMargin}
{/if} + + {#if !s3StorageConfigured && extra['x-no-s3-storage-workspace-warning']} + + {/if}